From 103a9f4466986ef57fc4f3f15dea95866bdead3f Mon Sep 17 00:00:00 2001
From: Eugen Rochko <eugen@zeonfederated.com>
Date: Sun, 16 Jun 2019 21:46:36 +0200
Subject: Fix sanitizer making block level elements unreadable (#10836)

Fix #10834
---
 spec/lib/sanitize_config_spec.rb | 26 ++++++++++++++++++++++++++
 1 file changed, 26 insertions(+)
 create mode 100644 spec/lib/sanitize_config_spec.rb

(limited to 'spec')

diff --git a/spec/lib/sanitize_config_spec.rb b/spec/lib/sanitize_config_spec.rb
new file mode 100644
index 000000000..bb3cf6f0b
--- /dev/null
+++ b/spec/lib/sanitize_config_spec.rb
@@ -0,0 +1,26 @@
+# frozen_string_literal: true
+
+require 'rails_helper'
+require Rails.root.join('app', 'lib', 'sanitize_config.rb')
+
+describe Sanitize::Config do
+  describe '::MASTODON_STRICT' do
+    subject { Sanitize::Config::MASTODON_STRICT }
+
+    it 'converts h1 to p' do
+      expect(Sanitize.fragment('<h1>Foo</h1>', subject)).to eq '<p>Foo</p>'
+    end
+
+    it 'converts ul to p' do
+      expect(Sanitize.fragment('<p>Check out:</p><ul><li>Foo</li><li>Bar</li></ul>', subject)).to eq '<p>Check out:</p><p>Foo<br>Bar</p>'
+    end
+
+    it 'converts p inside ul' do
+      expect(Sanitize.fragment('<ul><li><p>Foo</p><p>Bar</p></li><li>Baz</li></ul>', subject)).to eq '<p>Foo<br>Bar<br>Baz</p>'
+    end
+
+    it 'converts ul inside ul' do
+      expect(Sanitize.fragment('<ul><li>Foo</li><li><ul><li>Bar</li><li>Baz</li></ul></li></ul>', subject)).to eq '<p>Foo<br>Bar<br>Baz</p>'
+    end
+  end
+end
-- 
cgit 


From 7696f77245c2302787d239da50248385b3292a5e Mon Sep 17 00:00:00 2001
From: Eugen Rochko <eugen@zeonfederated.com>
Date: Thu, 20 Jun 2019 02:52:34 +0200
Subject: Add moderation API (#9387)

Fix #8580
Fix #7143
---
 app/controllers/admin/accounts_controller.rb       |   1 +
 .../api/v1/admin/account_actions_controller.rb     |  32 +++++
 .../api/v1/admin/accounts_controller.rb            | 128 ++++++++++++++++++
 app/controllers/api/v1/admin/reports_controller.rb | 108 +++++++++++++++
 app/models/account.rb                              |   2 +
 app/models/account_filter.rb                       |   2 +
 app/models/concerns/user_roles.rb                  |  14 ++
 app/models/report.rb                               |   3 +
 app/models/report_filter.rb                        |   2 +
 app/models/user.rb                                 |   1 +
 app/serializers/rest/admin/account_serializer.rb   |  77 +++++++++++
 app/serializers/rest/admin/report_serializer.rb    |  16 +++
 config/initializers/doorkeeper.rb                  |   8 +-
 config/locales/doorkeeper.en.yml                   |   6 +
 config/routes.rb                                   |  23 ++++
 .../v1/admin/account_actions_controller_spec.rb    |  57 ++++++++
 .../api/v1/admin/accounts_controller_spec.rb       | 147 +++++++++++++++++++++
 .../api/v1/admin/reports_controller_spec.rb        | 109 +++++++++++++++
 18 files changed, 735 insertions(+), 1 deletion(-)
 create mode 100644 app/controllers/api/v1/admin/account_actions_controller.rb
 create mode 100644 app/controllers/api/v1/admin/accounts_controller.rb
 create mode 100644 app/controllers/api/v1/admin/reports_controller.rb
 create mode 100644 app/serializers/rest/admin/account_serializer.rb
 create mode 100644 app/serializers/rest/admin/report_serializer.rb
 create mode 100644 spec/controllers/api/v1/admin/account_actions_controller_spec.rb
 create mode 100644 spec/controllers/api/v1/admin/accounts_controller_spec.rb
 create mode 100644 spec/controllers/api/v1/admin/reports_controller_spec.rb

(limited to 'spec')

diff --git a/app/controllers/admin/accounts_controller.rb b/app/controllers/admin/accounts_controller.rb
index b0d45ce47..0c7760d77 100644
--- a/app/controllers/admin/accounts_controller.rb
+++ b/app/controllers/admin/accounts_controller.rb
@@ -127,6 +127,7 @@ module Admin
         :by_domain,
         :active,
         :pending,
+        :disabled,
         :silenced,
         :suspended,
         :username,
diff --git a/app/controllers/api/v1/admin/account_actions_controller.rb b/app/controllers/api/v1/admin/account_actions_controller.rb
new file mode 100644
index 000000000..29c9b7107
--- /dev/null
+++ b/app/controllers/api/v1/admin/account_actions_controller.rb
@@ -0,0 +1,32 @@
+# frozen_string_literal: true
+
+class Api::V1::Admin::AccountActionsController < Api::BaseController
+  before_action -> { doorkeeper_authorize! :'admin:write', :'admin:write:accounts' }
+  before_action :require_staff!
+  before_action :set_account
+
+  def create
+    account_action                 = Admin::AccountAction.new(resource_params)
+    account_action.target_account  = @account
+    account_action.current_account = current_account
+    account_action.save!
+
+    render_empty
+  end
+
+  private
+
+  def set_account
+    @account = Account.find(params[:account_id])
+  end
+
+  def resource_params
+    params.permit(
+      :type,
+      :report_id,
+      :warning_preset_id,
+      :text,
+      :send_email_notification
+    )
+  end
+end
diff --git a/app/controllers/api/v1/admin/accounts_controller.rb b/app/controllers/api/v1/admin/accounts_controller.rb
new file mode 100644
index 000000000..c306180ca
--- /dev/null
+++ b/app/controllers/api/v1/admin/accounts_controller.rb
@@ -0,0 +1,128 @@
+# frozen_string_literal: true
+
+class Api::V1::Admin::AccountsController < Api::BaseController
+  include Authorization
+  include AccountableConcern
+
+  LIMIT = 100
+
+  before_action -> { doorkeeper_authorize! :'admin:read', :'admin:read:accounts' }, only: [:index, :show]
+  before_action -> { doorkeeper_authorize! :'admin:write', :'admin:write:accounts' }, except: [:index, :show]
+  before_action :require_staff!
+  before_action :set_accounts, only: :index
+  before_action :set_account, except: :index
+  before_action :require_local_account!, only: [:enable, :approve, :reject]
+
+  after_action :insert_pagination_headers, only: :index
+
+  FILTER_PARAMS = %i(
+    local
+    remote
+    by_domain
+    active
+    pending
+    disabled
+    silenced
+    suspended
+    username
+    display_name
+    email
+    ip
+    staff
+  ).freeze
+
+  PAGINATION_PARAMS = (%i(limit) + FILTER_PARAMS).freeze
+
+  def index
+    authorize :account, :index?
+    render json: @accounts, each_serializer: REST::Admin::AccountSerializer
+  end
+
+  def show
+    authorize @account, :show?
+    render json: @account, serializer: REST::Admin::AccountSerializer
+  end
+
+  def enable
+    authorize @account.user, :enable?
+    @account.user.enable!
+    log_action :enable, @account.user
+    render json: @account, serializer: REST::Admin::AccountSerializer
+  end
+
+  def approve
+    authorize @account.user, :approve?
+    @account.user.approve!
+    render json: @account, serializer: REST::Admin::AccountSerializer
+  end
+
+  def reject
+    authorize @account.user, :reject?
+    SuspendAccountService.new.call(@account, including_user: true, destroy: true, skip_distribution: true)
+    render json: @account, serializer: REST::Admin::AccountSerializer
+  end
+
+  def unsilence
+    authorize @account, :unsilence?
+    @account.unsilence!
+    log_action :unsilence, @account
+    render json: @account, serializer: REST::Admin::AccountSerializer
+  end
+
+  def unsuspend
+    authorize @account, :unsuspend?
+    @account.unsuspend!
+    log_action :unsuspend, @account
+    render json: @account, serializer: REST::Admin::AccountSerializer
+  end
+
+  private
+
+  def set_accounts
+    @accounts = filtered_accounts.order(id: :desc).includes(user: [:invite_request, :invite]).paginate_by_id(limit_param(LIMIT), params_slice(:max_id, :since_id, :min_id))
+  end
+
+  def set_account
+    @account = Account.find(params[:id])
+  end
+
+  def filtered_accounts
+    AccountFilter.new(filter_params).results
+  end
+
+  def filter_params
+    params.permit(*FILTER_PARAMS)
+  end
+
+  def insert_pagination_headers
+    set_pagination_headers(next_path, prev_path)
+  end
+
+  def next_path
+    api_v1_admin_accounts_url(pagination_params(max_id: pagination_max_id)) if records_continue?
+  end
+
+  def prev_path
+    api_v1_admin_accounts_url(pagination_params(min_id: pagination_since_id)) unless @accounts.empty?
+  end
+
+  def pagination_max_id
+    @accounts.last.id
+  end
+
+  def pagination_since_id
+    @accounts.first.id
+  end
+
+  def records_continue?
+    @accounts.size == limit_param(LIMIT)
+  end
+
+  def pagination_params(core_params)
+    params.slice(*PAGINATION_PARAMS).permit(*PAGINATION_PARAMS).merge(core_params)
+  end
+
+  def require_local_account!
+    forbidden unless @account.local? && @account.user.present?
+  end
+end
diff --git a/app/controllers/api/v1/admin/reports_controller.rb b/app/controllers/api/v1/admin/reports_controller.rb
new file mode 100644
index 000000000..1d48d3160
--- /dev/null
+++ b/app/controllers/api/v1/admin/reports_controller.rb
@@ -0,0 +1,108 @@
+# frozen_string_literal: true
+
+class Api::V1::Admin::ReportsController < Api::BaseController
+  include Authorization
+  include AccountableConcern
+
+  LIMIT = 100
+
+  before_action -> { doorkeeper_authorize! :'admin:read', :'admin:read:reports' }, only: [:index, :show]
+  before_action -> { doorkeeper_authorize! :'admin:write', :'admin:write:reports' }, except: [:index, :show]
+  before_action :require_staff!
+  before_action :set_reports, only: :index
+  before_action :set_report, except: :index
+
+  after_action :insert_pagination_headers, only: :index
+
+  FILTER_PARAMS = %i(
+    resolved
+    account_id
+    target_account_id
+  ).freeze
+
+  PAGINATION_PARAMS = (%i(limit) + FILTER_PARAMS).freeze
+
+  def index
+    authorize :report, :index?
+    render json: @reports, each_serializer: REST::Admin::ReportSerializer
+  end
+
+  def show
+    authorize @report, :show?
+    render json: @report, serializer: REST::Admin::ReportSerializer
+  end
+
+  def assign_to_self
+    authorize @report, :update?
+    @report.update!(assigned_account_id: current_account.id)
+    log_action :assigned_to_self, @report
+    render json: @report, serializer: REST::Admin::ReportSerializer
+  end
+
+  def unassign
+    authorize @report, :update?
+    @report.update!(assigned_account_id: nil)
+    log_action :unassigned, @report
+    render json: @report, serializer: REST::Admin::ReportSerializer
+  end
+
+  def reopen
+    authorize @report, :update?
+    @report.unresolve!
+    log_action :reopen, @report
+    render json: @report, serializer: REST::Admin::ReportSerializer
+  end
+
+  def resolve
+    authorize @report, :update?
+    @report.resolve!(current_account)
+    log_action :resolve, @report
+    render json: @report, serializer: REST::Admin::ReportSerializer
+  end
+
+  private
+
+  def set_reports
+    @reports = filtered_reports.order(id: :desc).with_accounts.paginate_by_id(limit_param(LIMIT), params_slice(:max_id, :since_id, :min_id))
+  end
+
+  def set_report
+    @report = Report.find(params[:id])
+  end
+
+  def filtered_reports
+    ReportFilter.new(filter_params).results
+  end
+
+  def filter_params
+    params.permit(*FILTER_PARAMS)
+  end
+
+  def insert_pagination_headers
+    set_pagination_headers(next_path, prev_path)
+  end
+
+  def next_path
+    api_v1_admin_reports_url(pagination_params(max_id: pagination_max_id)) if records_continue?
+  end
+
+  def prev_path
+    api_v1_admin_reports_url(pagination_params(min_id: pagination_since_id)) unless @reports.empty?
+  end
+
+  def pagination_max_id
+    @reports.last.id
+  end
+
+  def pagination_since_id
+    @reports.first.id
+  end
+
+  def records_continue?
+    @reports.size == limit_param(LIMIT)
+  end
+
+  def pagination_params(core_params)
+    params.slice(*PAGINATION_PARAMS).permit(*PAGINATION_PARAMS).merge(core_params)
+  end
+end
diff --git a/app/models/account.rb b/app/models/account.rb
index c977f887c..9276aa927 100644
--- a/app/models/account.rb
+++ b/app/models/account.rb
@@ -106,6 +106,8 @@ class Account < ApplicationRecord
            :confirmed?,
            :approved?,
            :pending?,
+           :disabled?,
+           :role,
            :admin?,
            :moderator?,
            :staff?,
diff --git a/app/models/account_filter.rb b/app/models/account_filter.rb
index d2503100c..c3b1fe08d 100644
--- a/app/models/account_filter.rb
+++ b/app/models/account_filter.rb
@@ -37,6 +37,8 @@ class AccountFilter
       Account.without_suspended
     when 'pending'
       accounts_with_users.merge User.pending
+    when 'disabled'
+      accounts_with_users.merge User.disabled
     when 'silenced'
       Account.silenced
     when 'suspended'
diff --git a/app/models/concerns/user_roles.rb b/app/models/concerns/user_roles.rb
index 58dffdc46..a42b4a172 100644
--- a/app/models/concerns/user_roles.rb
+++ b/app/models/concerns/user_roles.rb
@@ -13,6 +13,20 @@ module UserRoles
     admin? || moderator?
   end
 
+  def role=(value)
+    case value
+    when 'admin'
+      self.admin     = true
+      self.moderator = false
+    when 'moderator'
+      self.admin     = false
+      self.moderator = true
+    else
+      self.admin     = false
+      self.moderator = false
+    end
+  end
+
   def role
     if admin?
       'admin'
diff --git a/app/models/report.rb b/app/models/report.rb
index 86c303798..5192ceef7 100644
--- a/app/models/report.rb
+++ b/app/models/report.rb
@@ -17,6 +17,8 @@
 #
 
 class Report < ApplicationRecord
+  include Paginable
+
   belongs_to :account
   belongs_to :target_account, class_name: 'Account'
   belongs_to :action_taken_by_account, class_name: 'Account', optional: true
@@ -26,6 +28,7 @@ class Report < ApplicationRecord
 
   scope :unresolved, -> { where(action_taken: false) }
   scope :resolved,   -> { where(action_taken: true) }
+  scope :with_accounts, -> { includes([:account, :target_account, :action_taken_by_account, :assigned_account].each_with_object({}) { |k, h| h[k] = { user: [:invite_request, :invite] } }) }
 
   validates :comment, length: { maximum: 1000 }
 
diff --git a/app/models/report_filter.rb b/app/models/report_filter.rb
index 56ab28df7..a392d60c3 100644
--- a/app/models/report_filter.rb
+++ b/app/models/report_filter.rb
@@ -9,9 +9,11 @@ class ReportFilter
 
   def results
     scope = Report.unresolved
+
     params.each do |key, value|
       scope = scope.merge scope_for(key, value)
     end
+
     scope
   end
 
diff --git a/app/models/user.rb b/app/models/user.rb
index 4abf124fc..50873dd01 100644
--- a/app/models/user.rb
+++ b/app/models/user.rb
@@ -87,6 +87,7 @@ class User < ApplicationRecord
   scope :approved, -> { where(approved: true) }
   scope :confirmed, -> { where.not(confirmed_at: nil) }
   scope :enabled, -> { where(disabled: false) }
+  scope :disabled, -> { where(disabled: true) }
   scope :inactive, -> { where(arel_table[:current_sign_in_at].lt(ACTIVE_DURATION.ago)) }
   scope :active, -> { confirmed.where(arel_table[:current_sign_in_at].gteq(ACTIVE_DURATION.ago)).joins(:account).where(accounts: { suspended_at: nil }) }
   scope :matches_email, ->(value) { where(arel_table[:email].matches("#{value}%")) }
diff --git a/app/serializers/rest/admin/account_serializer.rb b/app/serializers/rest/admin/account_serializer.rb
new file mode 100644
index 000000000..f579d3302
--- /dev/null
+++ b/app/serializers/rest/admin/account_serializer.rb
@@ -0,0 +1,77 @@
+# frozen_string_literal: true
+
+class REST::Admin::AccountSerializer < ActiveModel::Serializer
+  attributes :id, :username, :domain, :created_at,
+             :email, :ip, :role, :confirmed, :suspended,
+             :silenced, :disabled, :approved, :locale,
+             :invite_request
+
+  attribute :created_by_application_id, if: :created_by_application?
+  attribute :invited_by_account_id, if: :invited?
+
+  has_one :account, serializer: REST::AccountSerializer
+
+  def id
+    object.id.to_s
+  end
+
+  def email
+    object.user_email
+  end
+
+  def ip
+    object.user_current_sign_in_ip.to_s.presence
+  end
+
+  def role
+    object.user_role
+  end
+
+  def suspended
+    object.suspended?
+  end
+
+  def silenced
+    object.silenced?
+  end
+
+  def confirmed
+    object.user_confirmed?
+  end
+
+  def disabled
+    object.user_disabled?
+  end
+
+  def approved
+    object.user_approved?
+  end
+
+  def account
+    object
+  end
+
+  def locale
+    object.user_locale
+  end
+
+  def created_by_application_id
+    object.user&.created_by_application_id&.to_s&.presence
+  end
+
+  def invite_request
+    object.user&.invite_request&.text
+  end
+
+  def invited_by_account_id
+    object.user&.invite&.user&.account_id&.to_s&.presence
+  end
+
+  def invited?
+    object.user&.invited?
+  end
+
+  def created_by_application?
+    object.user&.created_by_application_id&.present?
+  end
+end
diff --git a/app/serializers/rest/admin/report_serializer.rb b/app/serializers/rest/admin/report_serializer.rb
new file mode 100644
index 000000000..7a77132c0
--- /dev/null
+++ b/app/serializers/rest/admin/report_serializer.rb
@@ -0,0 +1,16 @@
+# frozen_string_literal: true
+
+class REST::Admin::ReportSerializer < ActiveModel::Serializer
+  attributes :id, :action_taken, :comment, :created_at, :updated_at
+
+  has_one :account, serializer: REST::Admin::AccountSerializer
+  has_one :target_account, serializer: REST::Admin::AccountSerializer
+  has_one :assigned_account, serializer: REST::Admin::AccountSerializer
+  has_one :action_taken_by_account, serializer: REST::Admin::AccountSerializer
+
+  has_many :statuses, serializer: REST::StatusSerializer
+
+  def id
+    object.id.to_s
+  end
+end
diff --git a/config/initializers/doorkeeper.rb b/config/initializers/doorkeeper.rb
index 367eead6a..914b3c001 100644
--- a/config/initializers/doorkeeper.rb
+++ b/config/initializers/doorkeeper.rb
@@ -80,7 +80,13 @@ Doorkeeper.configure do
                   :'read:search',
                   :'read:statuses',
                   :follow,
-                  :push
+                  :push,
+                  :'admin:read',
+                  :'admin:read:accounts',
+                  :'admin:read:reports',
+                  :'admin:write',
+                  :'admin:write:accounts',
+                  :'admin:write:reports'
 
   # Change the way client credentials are retrieved from the request object.
   # By default it retrieves first from the `HTTP_AUTHORIZATION` header, then
diff --git a/config/locales/doorkeeper.en.yml b/config/locales/doorkeeper.en.yml
index f1fe03716..d9b7c2c8e 100644
--- a/config/locales/doorkeeper.en.yml
+++ b/config/locales/doorkeeper.en.yml
@@ -114,6 +114,12 @@ en:
       application:
         title: OAuth authorization required
     scopes:
+      admin:read: read all data on the server
+      admin:read:accounts: read sensitive information of all accounts
+      admin:read:reports: read sensitive information of all reports and reported accounts
+      admin:write: modify all data on the server
+      admin:write:accounts: perform moderation actions on accounts
+      admin:write:reports: perform moderation actions on reports
       follow: modify account relationships
       push: receive your push notifications
       read: read all your account's data
diff --git a/config/routes.rb b/config/routes.rb
index 145079c69..764db8db2 100644
--- a/config/routes.rb
+++ b/config/routes.rb
@@ -398,6 +398,29 @@ Rails.application.routes.draw do
       namespace :push do
         resource :subscription, only: [:create, :show, :update, :destroy]
       end
+
+      namespace :admin do
+        resources :accounts, only: [:index, :show] do
+          member do
+            post :enable
+            post :unsilence
+            post :unsuspend
+            post :approve
+            post :reject
+          end
+
+          resource :action, only: [:create], controller: 'account_actions'
+        end
+
+        resources :reports, only: [:index, :show] do
+          member do
+            post :assign_to_self
+            post :unassign
+            post :reopen
+            post :resolve
+          end
+        end
+      end
     end
 
     namespace :v2 do
diff --git a/spec/controllers/api/v1/admin/account_actions_controller_spec.rb b/spec/controllers/api/v1/admin/account_actions_controller_spec.rb
new file mode 100644
index 000000000..a5a8f4bb0
--- /dev/null
+++ b/spec/controllers/api/v1/admin/account_actions_controller_spec.rb
@@ -0,0 +1,57 @@
+require 'rails_helper'
+
+RSpec.describe Api::V1::Admin::AccountActionsController, type: :controller do
+  render_views
+
+  let(:role)   { 'moderator' }
+  let(:user)   { Fabricate(:user, role: role, account: Fabricate(:account, username: 'alice')) }
+  let(:scopes) { 'admin:read admin:write' }
+  let(:token)  { Fabricate(:accessible_access_token, resource_owner_id: user.id, scopes: scopes) }
+  let(:account) { Fabricate(:user).account }
+
+  before do
+    allow(controller).to receive(:doorkeeper_token) { token }
+  end
+
+  shared_examples 'forbidden for wrong scope' do |wrong_scope|
+    let(:scopes) { wrong_scope }
+
+    it 'returns http forbidden' do
+      expect(response).to have_http_status(403)
+    end
+  end
+
+  shared_examples 'forbidden for wrong role' do |wrong_role|
+    let(:role) { wrong_role }
+
+    it 'returns http forbidden' do
+      expect(response).to have_http_status(403)
+    end
+  end
+
+  describe 'POST #create' do
+    before do
+      post :create, params: { account_id: account.id, type: 'disable' }
+    end
+
+    it_behaves_like 'forbidden for wrong scope', 'write:statuses'
+    it_behaves_like 'forbidden for wrong role', 'user'
+
+    it 'returns http success' do
+      expect(response).to have_http_status(200)
+    end
+
+    it 'performs action against account' do
+      expect(account.reload.user_disabled?).to be true
+    end
+
+    it 'logs action' do
+      log_item = Admin::ActionLog.last
+
+      expect(log_item).to_not be_nil
+      expect(log_item.action).to eq :disable
+      expect(log_item.account_id).to eq user.account_id
+      expect(log_item.target_id).to eq account.user.id
+    end
+  end
+end
diff --git a/spec/controllers/api/v1/admin/accounts_controller_spec.rb b/spec/controllers/api/v1/admin/accounts_controller_spec.rb
new file mode 100644
index 000000000..f3f9946ba
--- /dev/null
+++ b/spec/controllers/api/v1/admin/accounts_controller_spec.rb
@@ -0,0 +1,147 @@
+require 'rails_helper'
+
+RSpec.describe Api::V1::Admin::AccountsController, type: :controller do
+  render_views
+
+  let(:role)   { 'moderator' }
+  let(:user)   { Fabricate(:user, role: role, account: Fabricate(:account, username: 'alice')) }
+  let(:scopes) { 'admin:read admin:write' }
+  let(:token)  { Fabricate(:accessible_access_token, resource_owner_id: user.id, scopes: scopes) }
+  let(:account) { Fabricate(:user).account }
+
+  before do
+    allow(controller).to receive(:doorkeeper_token) { token }
+  end
+
+  shared_examples 'forbidden for wrong scope' do |wrong_scope|
+    let(:scopes) { wrong_scope }
+
+    it 'returns http forbidden' do
+      expect(response).to have_http_status(403)
+    end
+  end
+
+  shared_examples 'forbidden for wrong role' do |wrong_role|
+    let(:role) { wrong_role }
+
+    it 'returns http forbidden' do
+      expect(response).to have_http_status(403)
+    end
+  end
+
+  describe 'GET #index' do
+    before do
+      get :index
+    end
+
+    it_behaves_like 'forbidden for wrong scope', 'write:statuses'
+    it_behaves_like 'forbidden for wrong role', 'user'
+
+    it 'returns http success' do
+      expect(response).to have_http_status(200)
+    end
+  end
+
+  describe 'GET #show' do
+    before do
+      get :show, params: { id: account.id }
+    end
+
+    it_behaves_like 'forbidden for wrong scope', 'write:statuses'
+    it_behaves_like 'forbidden for wrong role', 'user'
+
+    it 'returns http success' do
+      expect(response).to have_http_status(200)
+    end
+  end
+
+  describe 'POST #approve' do
+    before do
+      account.user.update(approved: false)
+      post :approve, params: { id: account.id }
+    end
+
+    it_behaves_like 'forbidden for wrong scope', 'write:statuses'
+    it_behaves_like 'forbidden for wrong role', 'user'
+
+    it 'returns http success' do
+      expect(response).to have_http_status(200)
+    end
+
+    it 'approves user' do
+      expect(account.reload.user_approved?).to be true
+    end
+  end
+
+  describe 'POST #reject' do
+    before do
+      account.user.update(approved: false)
+      post :reject, params: { id: account.id }
+    end
+
+    it_behaves_like 'forbidden for wrong scope', 'write:statuses'
+    it_behaves_like 'forbidden for wrong role', 'user'
+
+    it 'returns http success' do
+      expect(response).to have_http_status(200)
+    end
+
+    it 'removes user' do
+      expect(User.where(id: account.user.id).count).to eq 0
+    end
+  end
+
+  describe 'POST #enable' do
+    before do
+      account.user.update(disabled: true)
+      post :enable, params: { id: account.id }
+    end
+
+    it_behaves_like 'forbidden for wrong scope', 'write:statuses'
+    it_behaves_like 'forbidden for wrong role', 'user'
+
+    it 'returns http success' do
+      expect(response).to have_http_status(200)
+    end
+
+    it 'enables user' do
+      expect(account.reload.user_disabled?).to be false
+    end
+  end
+
+  describe 'POST #unsuspend' do
+    before do
+      account.touch(:suspended_at)
+      post :unsuspend, params: { id: account.id }
+    end
+
+    it_behaves_like 'forbidden for wrong scope', 'write:statuses'
+    it_behaves_like 'forbidden for wrong role', 'user'
+
+    it 'returns http success' do
+      expect(response).to have_http_status(200)
+    end
+
+    it 'unsuspends account' do
+      expect(account.reload.suspended?).to be false
+    end
+  end
+
+  describe 'POST #unsilence' do
+    before do
+      account.touch(:silenced_at)
+      post :unsilence, params: { id: account.id }
+    end
+
+    it_behaves_like 'forbidden for wrong scope', 'write:statuses'
+    it_behaves_like 'forbidden for wrong role', 'user'
+
+    it 'returns http success' do
+      expect(response).to have_http_status(200)
+    end
+
+    it 'unsilences account' do
+      expect(account.reload.silenced?).to be false
+    end
+  end
+end
diff --git a/spec/controllers/api/v1/admin/reports_controller_spec.rb b/spec/controllers/api/v1/admin/reports_controller_spec.rb
new file mode 100644
index 000000000..4ed3c5dc4
--- /dev/null
+++ b/spec/controllers/api/v1/admin/reports_controller_spec.rb
@@ -0,0 +1,109 @@
+require 'rails_helper'
+
+RSpec.describe Api::V1::Admin::ReportsController, type: :controller do
+  render_views
+
+  let(:role)   { 'moderator' }
+  let(:user)   { Fabricate(:user, role: role, account: Fabricate(:account, username: 'alice')) }
+  let(:scopes) { 'admin:read admin:write' }
+  let(:token)  { Fabricate(:accessible_access_token, resource_owner_id: user.id, scopes: scopes) }
+  let(:report) { Fabricate(:report) }
+
+  before do
+    allow(controller).to receive(:doorkeeper_token) { token }
+  end
+
+  shared_examples 'forbidden for wrong scope' do |wrong_scope|
+    let(:scopes) { wrong_scope }
+
+    it 'returns http forbidden' do
+      expect(response).to have_http_status(403)
+    end
+  end
+
+  shared_examples 'forbidden for wrong role' do |wrong_role|
+    let(:role) { wrong_role }
+
+    it 'returns http forbidden' do
+      expect(response).to have_http_status(403)
+    end
+  end
+
+  describe 'GET #index' do
+    before do
+      get :index
+    end
+
+    it_behaves_like 'forbidden for wrong scope', 'write:statuses'
+    it_behaves_like 'forbidden for wrong role', 'user'
+
+    it 'returns http success' do
+      expect(response).to have_http_status(200)
+    end
+  end
+
+  describe 'GET #show' do
+    before do
+      get :show, params: { id: report.id }
+    end
+
+    it_behaves_like 'forbidden for wrong scope', 'write:statuses'
+    it_behaves_like 'forbidden for wrong role', 'user'
+
+    it 'returns http success' do
+      expect(response).to have_http_status(200)
+    end
+  end
+
+  describe 'POST #resolve' do
+    before do
+      post :resolve, params: { id: report.id }
+    end
+
+    it_behaves_like 'forbidden for wrong scope', 'write:statuses'
+    it_behaves_like 'forbidden for wrong role', 'user'
+
+    it 'returns http success' do
+      expect(response).to have_http_status(200)
+    end
+  end
+
+  describe 'POST #reopen' do
+    before do
+      post :reopen, params: { id: report.id }
+    end
+
+    it_behaves_like 'forbidden for wrong scope', 'write:statuses'
+    it_behaves_like 'forbidden for wrong role', 'user'
+
+    it 'returns http success' do
+      expect(response).to have_http_status(200)
+    end
+  end
+
+  describe 'POST #assign_to_self' do
+    before do
+      post :assign_to_self, params: { id: report.id }
+    end
+
+    it_behaves_like 'forbidden for wrong scope', 'write:statuses'
+    it_behaves_like 'forbidden for wrong role', 'user'
+
+    it 'returns http success' do
+      expect(response).to have_http_status(200)
+    end
+  end
+
+  describe 'POST #unassign' do
+    before do
+      post :unassign, params: { id: report.id }
+    end
+
+    it_behaves_like 'forbidden for wrong scope', 'write:statuses'
+    it_behaves_like 'forbidden for wrong role', 'user'
+
+    it 'returns http success' do
+      expect(response).to have_http_status(200)
+    end
+  end
+end
-- 
cgit 


From 707ddf7808f90e3ab042d7642d368c2ce8e95e6f Mon Sep 17 00:00:00 2001
From: Eugen Rochko <eugen@zeonfederated.com>
Date: Sat, 22 Jun 2019 00:13:10 +0200
Subject: Change domain blocks to automatically support subdomains (#11138)

* Change domain blocks to automatically support subdomains

If a more authoritative domain is blocked (example.com), then the
same block will be applied to a subdomain (foo.example.com)

* Match subdomains of existing accounts when blocking/unblocking domains

* Improve code style
---
 app/controllers/admin/domain_blocks_controller.rb  |  2 +-
 app/controllers/admin/instances_controller.rb      |  2 +-
 app/controllers/media_proxy_controller.rb          |  2 +-
 app/lib/activitypub/activity/create.rb             |  2 +-
 app/lib/activitypub/activity/flag.rb               |  2 +-
 app/lib/ostatus/activity/creation.rb               |  4 +--
 app/models/account.rb                              |  1 +
 app/models/custom_emoji.rb                         |  1 +
 app/models/domain_block.rb                         | 33 ++++++++++++++++++++--
 app/models/instance.rb                             |  2 +-
 .../activitypub/process_account_service.rb         |  2 +-
 app/services/block_domain_service.rb               |  4 +--
 app/services/resolve_account_service.rb            |  2 +-
 app/services/unblock_domain_service.rb             |  3 +-
 app/services/update_remote_profile_service.rb      |  4 +--
 spec/models/account_spec.rb                        | 17 +++++++++++
 spec/models/domain_block_spec.rb                   | 31 +++++++++++++++-----
 17 files changed, 89 insertions(+), 25 deletions(-)

(limited to 'spec')

diff --git a/app/controllers/admin/domain_blocks_controller.rb b/app/controllers/admin/domain_blocks_controller.rb
index 71597763b..377cac8ad 100644
--- a/app/controllers/admin/domain_blocks_controller.rb
+++ b/app/controllers/admin/domain_blocks_controller.rb
@@ -13,7 +13,7 @@ module Admin
       authorize :domain_block, :create?
 
       @domain_block = DomainBlock.new(resource_params)
-      existing_domain_block = resource_params[:domain].present? ? DomainBlock.find_by(domain: resource_params[:domain]) : nil
+      existing_domain_block = resource_params[:domain].present? ? DomainBlock.rule_for(resource_params[:domain]) : nil
 
       if existing_domain_block.present? && !@domain_block.stricter_than?(existing_domain_block)
         @domain_block.save
diff --git a/app/controllers/admin/instances_controller.rb b/app/controllers/admin/instances_controller.rb
index 6dd659a30..7888e844f 100644
--- a/app/controllers/admin/instances_controller.rb
+++ b/app/controllers/admin/instances_controller.rb
@@ -18,7 +18,7 @@ module Admin
       @blocks_count    = Block.where(target_account: Account.where(domain: params[:id])).count
       @available       = DeliveryFailureTracker.available?(Account.select(:shared_inbox_url).where(domain: params[:id]).first&.shared_inbox_url)
       @media_storage   = MediaAttachment.where(account: Account.where(domain: params[:id])).sum(:file_file_size)
-      @domain_block    = DomainBlock.find_by(domain: params[:id])
+      @domain_block    = DomainBlock.rule_for(params[:id])
     end
 
     private
diff --git a/app/controllers/media_proxy_controller.rb b/app/controllers/media_proxy_controller.rb
index 950cf6d09..8fc18dd06 100644
--- a/app/controllers/media_proxy_controller.rb
+++ b/app/controllers/media_proxy_controller.rb
@@ -39,6 +39,6 @@ class MediaProxyController < ApplicationController
   end
 
   def reject_media?
-    DomainBlock.find_by(domain: @media_attachment.account.domain)&.reject_media?
+    DomainBlock.reject_media?(@media_attachment.account.domain)
   end
 end
diff --git a/app/lib/activitypub/activity/create.rb b/app/lib/activitypub/activity/create.rb
index f55dd35b2..487e8e91e 100644
--- a/app/lib/activitypub/activity/create.rb
+++ b/app/lib/activitypub/activity/create.rb
@@ -380,7 +380,7 @@ class ActivityPub::Activity::Create < ActivityPub::Activity
 
   def skip_download?
     return @skip_download if defined?(@skip_download)
-    @skip_download ||= DomainBlock.find_by(domain: @account.domain)&.reject_media?
+    @skip_download ||= DomainBlock.reject_media?(@account.domain)
   end
 
   def reply_to_local?
diff --git a/app/lib/activitypub/activity/flag.rb b/app/lib/activitypub/activity/flag.rb
index f73b93058..1659bc61f 100644
--- a/app/lib/activitypub/activity/flag.rb
+++ b/app/lib/activitypub/activity/flag.rb
@@ -23,7 +23,7 @@ class ActivityPub::Activity::Flag < ActivityPub::Activity
   private
 
   def skip_reports?
-    DomainBlock.find_by(domain: @account.domain)&.reject_reports?
+    DomainBlock.reject_reports?(@account.domain)
   end
 
   def object_uris
diff --git a/app/lib/ostatus/activity/creation.rb b/app/lib/ostatus/activity/creation.rb
index 3840c8fbf..60de712db 100644
--- a/app/lib/ostatus/activity/creation.rb
+++ b/app/lib/ostatus/activity/creation.rb
@@ -148,7 +148,7 @@ class OStatus::Activity::Creation < OStatus::Activity::Base
   end
 
   def save_media
-    do_not_download = DomainBlock.find_by(domain: @account.domain)&.reject_media?
+    do_not_download = DomainBlock.reject_media?(@account.domain)
     media_attachments = []
 
     @xml.xpath('./xmlns:link[@rel="enclosure"]', xmlns: OStatus::TagManager::XMLNS).each do |link|
@@ -176,7 +176,7 @@ class OStatus::Activity::Creation < OStatus::Activity::Base
   end
 
   def save_emojis(parent)
-    do_not_download = DomainBlock.find_by(domain: parent.account.domain)&.reject_media?
+    do_not_download = DomainBlock.reject_media?(parent.account.domain)
 
     return if do_not_download
 
diff --git a/app/models/account.rb b/app/models/account.rb
index 9276aa927..c588451fc 100644
--- a/app/models/account.rb
+++ b/app/models/account.rb
@@ -98,6 +98,7 @@ class Account < ApplicationRecord
   scope :tagged_with, ->(tag) { joins(:accounts_tags).where(accounts_tags: { tag_id: tag }) }
   scope :by_recent_status, -> { order(Arel.sql('(case when account_stats.last_status_at is null then 1 else 0 end) asc, account_stats.last_status_at desc')) }
   scope :popular, -> { order('account_stats.followers_count desc') }
+  scope :by_domain_and_subdomains, ->(domain) { where(domain: domain).or(where(arel_table[:domain].matches('%.' + domain))) }
 
   delegate :email,
            :unconfirmed_email,
diff --git a/app/models/custom_emoji.rb b/app/models/custom_emoji.rb
index d3cc70504..e73cd9bd2 100644
--- a/app/models/custom_emoji.rb
+++ b/app/models/custom_emoji.rb
@@ -39,6 +39,7 @@ class CustomEmoji < ApplicationRecord
   scope :local,      -> { where(domain: nil) }
   scope :remote,     -> { where.not(domain: nil) }
   scope :alphabetic, -> { order(domain: :asc, shortcode: :asc) }
+  scope :by_domain_and_subdomains, ->(domain) { where(domain: domain).or(where(arel_table[:domain].matches('%.' + domain))) }
 
   remotable_attachment :image, LIMIT
 
diff --git a/app/models/domain_block.rb b/app/models/domain_block.rb
index 84c08c158..25d3b87ef 100644
--- a/app/models/domain_block.rb
+++ b/app/models/domain_block.rb
@@ -24,14 +24,41 @@ class DomainBlock < ApplicationRecord
 
   scope :matches_domain, ->(value) { where(arel_table[:domain].matches("%#{value}%")) }
 
-  def self.blocked?(domain)
-    where(domain: domain, severity: :suspend).exists?
+  class << self
+    def suspend?(domain)
+      !!rule_for(domain)&.suspend?
+    end
+
+    def silence?(domain)
+      !!rule_for(domain)&.silence?
+    end
+
+    def reject_media?(domain)
+      !!rule_for(domain)&.reject_media?
+    end
+
+    def reject_reports?(domain)
+      !!rule_for(domain)&.reject_reports?
+    end
+
+    alias blocked? suspend?
+
+    def rule_for(domain)
+      return if domain.blank?
+
+      uri      = Addressable::URI.new.tap { |u| u.host = domain.gsub(/[\/]/, '') }
+      segments = uri.normalized_host.split('.')
+      variants = segments.map.with_index { |_, i| segments[i..-1].join('.') }
+
+      where(domain: variants[0..-2]).order(Arel.sql('char_length(domain) desc')).first
+    end
   end
 
   def stricter_than?(other_block)
-    return true if suspend?
+    return true  if suspend?
     return false if other_block.suspend? && (silence? || noop?)
     return false if other_block.silence? && noop?
+
     (reject_media || !other_block.reject_media) && (reject_reports || !other_block.reject_reports)
   end
 
diff --git a/app/models/instance.rb b/app/models/instance.rb
index 7bf000d40..a01db1212 100644
--- a/app/models/instance.rb
+++ b/app/models/instance.rb
@@ -8,7 +8,7 @@ class Instance
   def initialize(resource)
     @domain         = resource.domain
     @accounts_count = resource.is_a?(DomainBlock) ? nil : resource.accounts_count
-    @domain_block   = resource.is_a?(DomainBlock) ? resource : DomainBlock.find_by(domain: domain)
+    @domain_block   = resource.is_a?(DomainBlock) ? resource : DomainBlock.rule_for(domain)
   end
 
   def cached_sample_accounts
diff --git a/app/services/activitypub/process_account_service.rb b/app/services/activitypub/process_account_service.rb
index ad22d37fe..05c017bdf 100644
--- a/app/services/activitypub/process_account_service.rb
+++ b/app/services/activitypub/process_account_service.rb
@@ -205,7 +205,7 @@ class ActivityPub::ProcessAccountService < BaseService
 
   def domain_block
     return @domain_block if defined?(@domain_block)
-    @domain_block = DomainBlock.find_by(domain: @domain)
+    @domain_block = DomainBlock.rule_for(@domain)
   end
 
   def key_changed?
diff --git a/app/services/block_domain_service.rb b/app/services/block_domain_service.rb
index 497f0394b..c6eef04d4 100644
--- a/app/services/block_domain_service.rb
+++ b/app/services/block_domain_service.rb
@@ -76,7 +76,7 @@ class BlockDomainService < BaseService
   end
 
   def blocked_domain_accounts
-    Account.where(domain: blocked_domain)
+    Account.by_domain_and_subdomains(blocked_domain)
   end
 
   def media_from_blocked_domain
@@ -84,6 +84,6 @@ class BlockDomainService < BaseService
   end
 
   def emojis_from_blocked_domains
-    CustomEmoji.where(domain: blocked_domain)
+    CustomEmoji.by_domain_and_subdomains(blocked_domain)
   end
 end
diff --git a/app/services/resolve_account_service.rb b/app/services/resolve_account_service.rb
index 11e33a83a..57c9ccfe1 100644
--- a/app/services/resolve_account_service.rb
+++ b/app/services/resolve_account_service.rb
@@ -146,7 +146,7 @@ class ResolveAccountService < BaseService
 
   def domain_block
     return @domain_block if defined?(@domain_block)
-    @domain_block = DomainBlock.find_by(domain: @domain)
+    @domain_block = DomainBlock.rule_for(@domain)
   end
 
   def atom_url
diff --git a/app/services/unblock_domain_service.rb b/app/services/unblock_domain_service.rb
index 9b8526fbe..fc262a50a 100644
--- a/app/services/unblock_domain_service.rb
+++ b/app/services/unblock_domain_service.rb
@@ -14,7 +14,8 @@ class UnblockDomainService < BaseService
   end
 
   def blocked_accounts
-    scope = Account.where(domain: domain_block.domain)
+    scope = Account.by_domain_and_subdomains(domain_block.domain)
+
     if domain_block.silence?
       scope.where(silenced_at: @domain_block.created_at)
     else
diff --git a/app/services/update_remote_profile_service.rb b/app/services/update_remote_profile_service.rb
index 68d36addf..403395a0d 100644
--- a/app/services/update_remote_profile_service.rb
+++ b/app/services/update_remote_profile_service.rb
@@ -26,7 +26,7 @@ class UpdateRemoteProfileService < BaseService
     account.note         = remote_profile.note         || ''
     account.locked       = remote_profile.locked?
 
-    if !account.suspended? && !DomainBlock.find_by(domain: account.domain)&.reject_media?
+    if !account.suspended? && !DomainBlock.reject_media?(account.domain)
       if remote_profile.avatar.present?
         account.avatar_remote_url = remote_profile.avatar
       else
@@ -46,7 +46,7 @@ class UpdateRemoteProfileService < BaseService
   end
 
   def save_emojis
-    do_not_download = DomainBlock.find_by(domain: account.domain)&.reject_media?
+    do_not_download = DomainBlock.reject_media?(account.domain)
 
     return if do_not_download
 
diff --git a/spec/models/account_spec.rb b/spec/models/account_spec.rb
index 379872316..ce9ea250d 100644
--- a/spec/models/account_spec.rb
+++ b/spec/models/account_spec.rb
@@ -687,6 +687,23 @@ RSpec.describe Account, type: :model do
       end
     end
 
+    describe 'by_domain_and_subdomains' do
+      it 'returns exact domain matches' do
+        account = Fabricate(:account, domain: 'example.com')
+        expect(Account.by_domain_and_subdomains('example.com')).to eq [account]
+      end
+
+      it 'returns subdomains' do
+        account = Fabricate(:account, domain: 'foo.example.com')
+        expect(Account.by_domain_and_subdomains('example.com')).to eq [account]
+      end
+
+      it 'does not return partially matching domains' do
+        account = Fabricate(:account, domain: 'grexample.com')
+        expect(Account.by_domain_and_subdomains('example.com')).to_not eq [account]
+      end
+    end
+
     describe 'expiring' do
       it 'returns remote accounts with followers whose subscription expiration date is past or not given' do
         local = Fabricate(:account, domain: nil)
diff --git a/spec/models/domain_block_spec.rb b/spec/models/domain_block_spec.rb
index 0035fd0ff..d98c5e118 100644
--- a/spec/models/domain_block_spec.rb
+++ b/spec/models/domain_block_spec.rb
@@ -21,23 +21,40 @@ RSpec.describe DomainBlock, type: :model do
     end
   end
 
-  describe 'blocked?' do
+  describe '.blocked?' do
     it 'returns true if the domain is suspended' do
-      Fabricate(:domain_block, domain: 'domain', severity: :suspend)
-      expect(DomainBlock.blocked?('domain')).to eq true
+      Fabricate(:domain_block, domain: 'example.com', severity: :suspend)
+      expect(DomainBlock.blocked?('example.com')).to eq true
     end
 
     it 'returns false even if the domain is silenced' do
-      Fabricate(:domain_block, domain: 'domain', severity: :silence)
-      expect(DomainBlock.blocked?('domain')).to eq false
+      Fabricate(:domain_block, domain: 'example.com', severity: :silence)
+      expect(DomainBlock.blocked?('example.com')).to eq false
     end
 
     it 'returns false if the domain is not suspended nor silenced' do
-      expect(DomainBlock.blocked?('domain')).to eq false
+      expect(DomainBlock.blocked?('example.com')).to eq false
     end
   end
 
-  describe 'stricter_than?' do
+  describe '.rule_for' do
+    it 'returns rule matching a blocked domain' do
+      block = Fabricate(:domain_block, domain: 'example.com')
+      expect(DomainBlock.rule_for('example.com')).to eq block
+    end
+
+    it 'returns a rule matching a subdomain of a blocked domain' do
+      block = Fabricate(:domain_block, domain: 'example.com')
+      expect(DomainBlock.rule_for('sub.example.com')).to eq block
+    end
+
+    it 'returns a rule matching a blocked subdomain' do
+      block = Fabricate(:domain_block, domain: 'sub.example.com')
+      expect(DomainBlock.rule_for('sub.example.com')).to eq block
+    end
+  end
+
+  describe '#stricter_than?' do
     it 'returns true if the new block has suspend severity while the old has lower severity' do
       suspend = DomainBlock.new(domain: 'domain', severity: :suspend)
       silence = DomainBlock.new(domain: 'domain', severity: :silence)
-- 
cgit 


From 47ef4a6c7a74072daff8b23c4af3e300bb75ba1a Mon Sep 17 00:00:00 2001
From: ThibG <thib@sitedethib.com>
Date: Tue, 25 Jun 2019 14:45:14 +0200
Subject: Apply filters to poll options (#11174)

* Apply filters to poll options in WebUI

Fixes #11128

* Apply filters to poll options server-side

* Add poll options to searchable text
---
 app/chewy/statuses_index.rb                            | 2 +-
 app/javascript/mastodon/actions/importer/normalizer.js | 2 +-
 app/lib/feed_manager.rb                                | 3 ++-
 spec/lib/feed_manager_spec.rb                          | 8 ++++++++
 4 files changed, 12 insertions(+), 3 deletions(-)

(limited to 'spec')

diff --git a/app/chewy/statuses_index.rb b/app/chewy/statuses_index.rb
index 8ce413f8a..f5983a5a5 100644
--- a/app/chewy/statuses_index.rb
+++ b/app/chewy/statuses_index.rb
@@ -51,7 +51,7 @@ class StatusesIndex < Chewy::Index
       field :id, type: 'long'
       field :account_id, type: 'long'
 
-      field :text, type: 'text', value: ->(status) { [status.spoiler_text, Formatter.instance.plaintext(status)].concat(status.media_attachments.map(&:description)).join("\n\n") } do
+      field :text, type: 'text', value: ->(status) { [status.spoiler_text, Formatter.instance.plaintext(status)].concat(status.media_attachments.map(&:description)).concat(status.preloadable_poll ? status_preloadable_poll.options : []).join("\n\n") } do
         field :stemmed, type: 'text', analyzer: 'content'
       end
 
diff --git a/app/javascript/mastodon/actions/importer/normalizer.js b/app/javascript/mastodon/actions/importer/normalizer.js
index 5badb0c49..b250ee076 100644
--- a/app/javascript/mastodon/actions/importer/normalizer.js
+++ b/app/javascript/mastodon/actions/importer/normalizer.js
@@ -56,7 +56,7 @@ export function normalizeStatus(status, normalOldStatus) {
     normalStatus.hidden = normalOldStatus.get('hidden');
   } else {
     const spoilerText   = normalStatus.spoiler_text || '';
-    const searchContent = [spoilerText, status.content].join('\n\n').replace(/<br\s*\/?>/g, '\n').replace(/<\/p><p>/g, '\n\n');
+    const searchContent = ([spoilerText, status.content].concat((status.poll && status.poll.options) ? status.poll.options.map(option => option.title) : [])).join('\n\n').replace(/<br\s*\/?>/g, '\n').replace(/<\/p><p>/g, '\n\n');
     const emojiMap      = makeEmojiMap(normalStatus);
 
     normalStatus.search_index = domParser.parseFromString(searchContent, 'text/html').documentElement.textContent;
diff --git a/app/lib/feed_manager.rb b/app/lib/feed_manager.rb
index d77cdb3a3..ed3ce6112 100644
--- a/app/lib/feed_manager.rb
+++ b/app/lib/feed_manager.rb
@@ -220,7 +220,8 @@ class FeedManager
     status         = status.reblog if status.reblog?
 
     !combined_regex.match(Formatter.instance.plaintext(status)).nil? ||
-      (status.spoiler_text.present? && !combined_regex.match(status.spoiler_text).nil?)
+      (status.spoiler_text.present? && !combined_regex.match(status.spoiler_text).nil?) ||
+      (status.preloadable_poll && !combined_regex.match(status.preloadable_poll.options.join("\n\n")).nil?)
   end
 
   # Adds a status to an account's feed, returning true if a status was
diff --git a/spec/lib/feed_manager_spec.rb b/spec/lib/feed_manager_spec.rb
index 5f8eb86a8..9bdb675e1 100644
--- a/spec/lib/feed_manager_spec.rb
+++ b/spec/lib/feed_manager_spec.rb
@@ -149,6 +149,14 @@ RSpec.describe FeedManager do
           status = Fabricate(:status, text: 'shiitake', account: jeff)
           expect(FeedManager.instance.filter?(:home, status, alice.id)).to be true
         end
+
+        it 'returns true if phrase is contained in a poll option' do
+          alice.custom_filters.create!(phrase: 'farts', context: %w(home public), irreversible: true)
+          alice.custom_filters.create!(phrase: 'pop tarts', context: %w(home), irreversible: true)
+          alice.follow!(jeff)
+          status = Fabricate(:status, text: 'what do you prefer', poll: Fabricate(:poll, options: %w(farts POP TARts)), account: jeff)
+          expect(FeedManager.instance.filter?(:home, status, alice.id)).to be true
+        end
       end
     end
 
-- 
cgit 


From 915c619394165a114a4ab316165aecac3386cf2f Mon Sep 17 00:00:00 2001
From: ThibG <thib@sitedethib.com>
Date: Wed, 26 Jun 2019 19:32:36 +0200
Subject: Add support for Audio activities (#11189)

Fixes #11127
---
 app/lib/activitypub/activity.rb                    |  2 +-
 app/services/resolve_url_service.rb                |  4 +--
 .../fetch_remote_status_service_spec.rb            | 33 ++++++++++++++++++++++
 3 files changed, 36 insertions(+), 3 deletions(-)

(limited to 'spec')

diff --git a/app/lib/activitypub/activity.rb b/app/lib/activitypub/activity.rb
index 66b5763a9..a4a9baaee 100644
--- a/app/lib/activitypub/activity.rb
+++ b/app/lib/activitypub/activity.rb
@@ -5,7 +5,7 @@ class ActivityPub::Activity
   include Redisable
 
   SUPPORTED_TYPES = %w(Note Question).freeze
-  CONVERTED_TYPES = %w(Image Video Article Page).freeze
+  CONVERTED_TYPES = %w(Image Audio Video Article Page).freeze
 
   def initialize(json, account, **options)
     @json    = json
diff --git a/app/services/resolve_url_service.rb b/app/services/resolve_url_service.rb
index b98759bf6..bbdc0a595 100644
--- a/app/services/resolve_url_service.rb
+++ b/app/services/resolve_url_service.rb
@@ -18,9 +18,9 @@ class ResolveURLService < BaseService
   private
 
   def process_url
-    if equals_or_includes_any?(type, %w(Application Group Organization Person Service))
+    if equals_or_includes_any?(type, ActivityPub::FetchRemoteAccountService::SUPPORTED_TYPES)
       FetchRemoteAccountService.new.call(atom_url, body, protocol)
-    elsif equals_or_includes_any?(type, %w(Note Article Image Video Page Question))
+    elsif equals_or_includes_any?(type, ActivityPub::Activity::Create::SUPPORTED_TYPES + ActivityPub::Activity::Create::CONVERTED_TYPES)
       FetchRemoteStatusService.new.call(atom_url, body, protocol)
     end
   end
diff --git a/spec/services/activitypub/fetch_remote_status_service_spec.rb b/spec/services/activitypub/fetch_remote_status_service_spec.rb
index 9ae409996..78dd59e3b 100644
--- a/spec/services/activitypub/fetch_remote_status_service_spec.rb
+++ b/spec/services/activitypub/fetch_remote_status_service_spec.rb
@@ -71,6 +71,39 @@ RSpec.describe ActivityPub::FetchRemoteStatusService, type: :service do
       end
     end
 
+    context 'with Audio object' do
+      let(:object) do
+        {
+          '@context': 'https://www.w3.org/ns/activitystreams',
+          id: "https://#{valid_domain}/@foo/1234",
+          type: 'Audio',
+          name: 'Nyan Cat 10 hours remix',
+          attributedTo: ActivityPub::TagManager.instance.uri_for(sender),
+          url: [
+            {
+              type: 'Link',
+              mimeType: 'application/x-bittorrent',
+              href: "https://#{valid_domain}/12345.torrent",
+            },
+
+            {
+              type: 'Link',
+              mimeType: 'text/html',
+              href: "https://#{valid_domain}/watch?v=12345",
+            },
+          ],
+        }
+      end
+
+      it 'creates status' do
+        status = sender.statuses.first
+
+        expect(status).to_not be_nil
+        expect(status.url).to eq "https://#{valid_domain}/watch?v=12345"
+        expect(strip_tags(status.text)).to eq "Nyan Cat 10 hours remix https://#{valid_domain}/watch?v=12345"
+      end
+    end
+
     context 'with wrong id' do
       let(:note) do
         {
-- 
cgit 


From 9a90ec3b3be655170077a18ad2fe81ade1498d76 Mon Sep 17 00:00:00 2001
From: ThibG <thib@sitedethib.com>
Date: Thu, 27 Jun 2019 19:41:55 +0200
Subject: Fix account URI in UpdatePollSerializer (#11194)

* Fix account URI in UpdatePollSerializer

Fixes #11185

* Add specs
---
 .../activitypub/update_poll_serializer.rb          |  2 +-
 spec/serializers/activitypub/update_poll_spec.rb   | 27 ++++++++++++++++++++++
 .../distribute_poll_update_worker_spec.rb          | 22 ++++++++++++++++++
 3 files changed, 50 insertions(+), 1 deletion(-)
 create mode 100644 spec/serializers/activitypub/update_poll_spec.rb
 create mode 100644 spec/workers/activitypub/distribute_poll_update_worker_spec.rb

(limited to 'spec')

diff --git a/app/serializers/activitypub/update_poll_serializer.rb b/app/serializers/activitypub/update_poll_serializer.rb
index b894f309f..1d47b9764 100644
--- a/app/serializers/activitypub/update_poll_serializer.rb
+++ b/app/serializers/activitypub/update_poll_serializer.rb
@@ -14,7 +14,7 @@ class ActivityPub::UpdatePollSerializer < ActivityPub::Serializer
   end
 
   def actor
-    ActivityPub::TagManager.instance.uri_for(object)
+    ActivityPub::TagManager.instance.uri_for(object.account)
   end
 
   def to
diff --git a/spec/serializers/activitypub/update_poll_spec.rb b/spec/serializers/activitypub/update_poll_spec.rb
new file mode 100644
index 000000000..f9e035eab
--- /dev/null
+++ b/spec/serializers/activitypub/update_poll_spec.rb
@@ -0,0 +1,27 @@
+# frozen_string_literal: true
+
+require 'rails_helper'
+
+describe ActivityPub::UpdatePollSerializer do
+  let(:account) { Fabricate(:account) }
+  let(:poll)    { Fabricate(:poll, account: account) }
+  let!(:status) { Fabricate(:status, account: account, poll: poll) }
+
+  before(:each) do
+    @serialization = ActiveModelSerializers::SerializableResource.new(status, serializer: ActivityPub::UpdatePollSerializer, adapter: ActivityPub::Adapter)
+  end
+
+  subject { JSON.parse(@serialization.to_json) }
+
+  it 'has a Update type' do
+    expect(subject['type']).to eql('Update')
+  end
+
+  it 'has an object with Question type' do
+    expect(subject['object']['type']).to eql('Question')
+  end
+
+  it 'has the correct actor URI set' do
+    expect(subject['actor']).to eql(ActivityPub::TagManager.instance.uri_for(account))
+  end
+end
diff --git a/spec/workers/activitypub/distribute_poll_update_worker_spec.rb b/spec/workers/activitypub/distribute_poll_update_worker_spec.rb
new file mode 100644
index 000000000..7eb6119fd
--- /dev/null
+++ b/spec/workers/activitypub/distribute_poll_update_worker_spec.rb
@@ -0,0 +1,22 @@
+require 'rails_helper'
+
+describe ActivityPub::DistributePollUpdateWorker do
+  subject { described_class.new }
+
+  let(:account)  { Fabricate(:account) }
+  let(:follower) { Fabricate(:account, protocol: :activitypub, inbox_url: 'http://example.com') }
+  let(:poll)     { Fabricate(:poll, account: account) }
+  let!(:status)  { Fabricate(:status, account: account, poll: poll) }
+
+  describe '#perform' do
+    before do
+      allow(ActivityPub::DeliveryWorker).to receive(:push_bulk)
+      follower.follow!(account)
+    end
+
+    it 'delivers to followers' do
+      subject.perform(status.id)
+      expect(ActivityPub::DeliveryWorker).to have_received(:push_bulk).with(['http://example.com'])
+    end
+  end
+end
-- 
cgit 


From e64e6a03dd1e0978fee48f0596dcfbc7fd29958f Mon Sep 17 00:00:00 2001
From: Eugen Rochko <eugen@zeonfederated.com>
Date: Fri, 28 Jun 2019 15:54:10 +0200
Subject: Add categories for custom emojis (#11196)

Fix #7940
---
 app/controllers/api/v1/custom_emojis_controller.rb |  2 +-
 .../compose/components/emoji_picker_dropdown.js    | 30 ++++++++++++----------
 app/javascript/mastodon/features/emoji/emoji.js    |  3 +++
 app/models/custom_emoji.rb                         |  2 ++
 app/models/custom_emoji_category.rb                | 15 +++++++++++
 app/serializers/rest/custom_emoji_serializer.rb    | 10 ++++++++
 ...0190627222225_create_custom_emoji_categories.rb |  9 +++++++
 ...90627222826_add_category_id_to_custom_emojis.rb |  5 ++++
 db/schema.rb                                       | 10 +++++++-
 lib/mastodon/emoji_cli.rb                          |  6 +++++
 .../custom_emoji_category_fabricator.rb            |  3 +++
 spec/models/custom_emoji_category_spec.rb          |  5 ++++
 yarn.lock                                          |  4 +--
 13 files changed, 86 insertions(+), 18 deletions(-)
 create mode 100644 app/models/custom_emoji_category.rb
 create mode 100644 db/migrate/20190627222225_create_custom_emoji_categories.rb
 create mode 100644 db/migrate/20190627222826_add_category_id_to_custom_emojis.rb
 create mode 100644 spec/fabricators/custom_emoji_category_fabricator.rb
 create mode 100644 spec/models/custom_emoji_category_spec.rb

(limited to 'spec')

diff --git a/app/controllers/api/v1/custom_emojis_controller.rb b/app/controllers/api/v1/custom_emojis_controller.rb
index 1bb19a09d..b6877fb3c 100644
--- a/app/controllers/api/v1/custom_emojis_controller.rb
+++ b/app/controllers/api/v1/custom_emojis_controller.rb
@@ -7,7 +7,7 @@ class Api::V1::CustomEmojisController < Api::BaseController
 
   def index
     render_cached_json('api:v1:custom_emojis', expires_in: 1.minute) do
-      ActiveModelSerializers::SerializableResource.new(CustomEmoji.local.where(disabled: false), each_serializer: REST::CustomEmojiSerializer)
+      ActiveModelSerializers::SerializableResource.new(CustomEmoji.local.where(disabled: false).includes(:category), each_serializer: REST::CustomEmojiSerializer)
     end
   end
 end
diff --git a/app/javascript/mastodon/features/compose/components/emoji_picker_dropdown.js b/app/javascript/mastodon/features/compose/components/emoji_picker_dropdown.js
index c1429c756..e57c3c20c 100644
--- a/app/javascript/mastodon/features/compose/components/emoji_picker_dropdown.js
+++ b/app/javascript/mastodon/features/compose/components/emoji_picker_dropdown.js
@@ -6,7 +6,7 @@ import Overlay from 'react-overlays/lib/Overlay';
 import classNames from 'classnames';
 import ImmutablePropTypes from 'react-immutable-proptypes';
 import detectPassiveEvents from 'detect-passive-events';
-import { buildCustomEmojis } from '../../emoji/emoji';
+import { buildCustomEmojis, categoriesFromEmojis } from '../../emoji/emoji';
 
 const messages = defineMessages({
   emoji: { id: 'emoji_button.label', defaultMessage: 'Insert emoji' },
@@ -31,19 +31,6 @@ let EmojiPicker, Emoji; // load asynchronously
 const backgroundImageFn = () => `${assetHost}/emoji/sheet_10.png`;
 const listenerOptions = detectPassiveEvents.hasSupport ? { passive: true } : false;
 
-const categoriesSort = [
-  'recent',
-  'custom',
-  'people',
-  'nature',
-  'foods',
-  'activity',
-  'places',
-  'objects',
-  'symbols',
-  'flags',
-];
-
 class ModifierPickerMenu extends React.PureComponent {
 
   static propTypes = {
@@ -241,8 +228,23 @@ class EmojiPickerMenu extends React.PureComponent {
     }
 
     const title = intl.formatMessage(messages.emoji);
+
     const { modifierOpen } = this.state;
 
+    const categoriesSort = [
+      'recent',
+      'people',
+      'nature',
+      'foods',
+      'activity',
+      'places',
+      'objects',
+      'symbols',
+      'flags',
+    ];
+
+    categoriesSort.splice(1, 0, ...Array.from(categoriesFromEmojis(custom_emojis)).sort());
+
     return (
       <div className={classNames('emoji-picker-dropdown__menu', { selecting: modifierOpen })} style={style} ref={this.setRef}>
         <EmojiPicker
diff --git a/app/javascript/mastodon/features/emoji/emoji.js b/app/javascript/mastodon/features/emoji/emoji.js
index 988cea253..01b5a6664 100644
--- a/app/javascript/mastodon/features/emoji/emoji.js
+++ b/app/javascript/mastodon/features/emoji/emoji.js
@@ -92,8 +92,11 @@ export const buildCustomEmojis = (customEmojis) => {
       keywords: [name],
       imageUrl: url,
       custom: true,
+      customCategory: emoji.get('category'),
     });
   });
 
   return emojis;
 };
+
+export const categoriesFromEmojis = customEmojis => customEmojis.reduce((set, emoji) => set.add(emoji.get('category') ? `custom-${emoji.get('category')}` : 'custom'), new Set());
diff --git a/app/models/custom_emoji.rb b/app/models/custom_emoji.rb
index e73cd9bd2..514cf4845 100644
--- a/app/models/custom_emoji.rb
+++ b/app/models/custom_emoji.rb
@@ -16,6 +16,7 @@
 #  uri                :string
 #  image_remote_url   :string
 #  visible_in_picker  :boolean          default(TRUE), not null
+#  category_id        :bigint(8)
 #
 
 class CustomEmoji < ApplicationRecord
@@ -27,6 +28,7 @@ class CustomEmoji < ApplicationRecord
     :(#{SHORTCODE_RE_FRAGMENT}):
     (?=[^[:alnum:]:]|$)/x
 
+  belongs_to :category, class_name: 'CustomEmojiCategory', optional: true
   has_one :local_counterpart, -> { where(domain: nil) }, class_name: 'CustomEmoji', primary_key: :shortcode, foreign_key: :shortcode
 
   has_attached_file :image, styles: { static: { format: 'png', convert_options: '-coalesce -strip' } }
diff --git a/app/models/custom_emoji_category.rb b/app/models/custom_emoji_category.rb
new file mode 100644
index 000000000..7d8c0ee2d
--- /dev/null
+++ b/app/models/custom_emoji_category.rb
@@ -0,0 +1,15 @@
+# frozen_string_literal: true
+
+# == Schema Information
+#
+# Table name: custom_emoji_categories
+#
+#  id         :bigint(8)        not null, primary key
+#  name       :string
+#  created_at :datetime         not null
+#  updated_at :datetime         not null
+#
+
+class CustomEmojiCategory < ApplicationRecord
+  has_many :emojis, class_name: 'CustomEmoji', foreign_key: 'category_id', inverse_of: :category
+end
diff --git a/app/serializers/rest/custom_emoji_serializer.rb b/app/serializers/rest/custom_emoji_serializer.rb
index 65686a866..aff58e4d9 100644
--- a/app/serializers/rest/custom_emoji_serializer.rb
+++ b/app/serializers/rest/custom_emoji_serializer.rb
@@ -5,6 +5,8 @@ class REST::CustomEmojiSerializer < ActiveModel::Serializer
 
   attributes :shortcode, :url, :static_url, :visible_in_picker
 
+  attribute :category, if: :category_loaded?
+
   def url
     full_asset_url(object.image.url)
   end
@@ -12,4 +14,12 @@ class REST::CustomEmojiSerializer < ActiveModel::Serializer
   def static_url
     full_asset_url(object.image.url(:static))
   end
+
+  def category
+    object.category.name
+  end
+
+  def category_loaded?
+    object.association(:category).loaded? && object.category.present?
+  end
 end
diff --git a/db/migrate/20190627222225_create_custom_emoji_categories.rb b/db/migrate/20190627222225_create_custom_emoji_categories.rb
new file mode 100644
index 000000000..4713793e6
--- /dev/null
+++ b/db/migrate/20190627222225_create_custom_emoji_categories.rb
@@ -0,0 +1,9 @@
+class CreateCustomEmojiCategories < ActiveRecord::Migration[5.2]
+  def change
+    create_table :custom_emoji_categories do |t|
+      t.string :name, index: { unique: true }
+
+      t.timestamps
+    end
+  end
+end
diff --git a/db/migrate/20190627222826_add_category_id_to_custom_emojis.rb b/db/migrate/20190627222826_add_category_id_to_custom_emojis.rb
new file mode 100644
index 000000000..873b4d05f
--- /dev/null
+++ b/db/migrate/20190627222826_add_category_id_to_custom_emojis.rb
@@ -0,0 +1,5 @@
+class AddCategoryIdToCustomEmojis < ActiveRecord::Migration[5.2]
+  def change
+    add_column :custom_emojis, :category_id, :bigint
+  end
+end
diff --git a/db/schema.rb b/db/schema.rb
index f633f4e3f..09e6c9fae 100644
--- a/db/schema.rb
+++ b/db/schema.rb
@@ -10,7 +10,7 @@
 #
 # It's strongly recommended that you check this file into your version control system.
 
-ActiveRecord::Schema.define(version: 2019_05_29_143559) do
+ActiveRecord::Schema.define(version: 2019_06_27_222826) do
 
   # These are extensions that must be enabled in order to support this database
   enable_extension "plpgsql"
@@ -208,6 +208,13 @@ ActiveRecord::Schema.define(version: 2019_05_29_143559) do
     t.index ["uri"], name: "index_conversations_on_uri", unique: true
   end
 
+  create_table "custom_emoji_categories", force: :cascade do |t|
+    t.string "name"
+    t.datetime "created_at", null: false
+    t.datetime "updated_at", null: false
+    t.index ["name"], name: "index_custom_emoji_categories_on_name", unique: true
+  end
+
   create_table "custom_emojis", force: :cascade do |t|
     t.string "shortcode", default: "", null: false
     t.string "domain"
@@ -221,6 +228,7 @@ ActiveRecord::Schema.define(version: 2019_05_29_143559) do
     t.string "uri"
     t.string "image_remote_url"
     t.boolean "visible_in_picker", default: true, null: false
+    t.bigint "category_id"
     t.index ["shortcode", "domain"], name: "index_custom_emojis_on_shortcode_and_domain", unique: true
   end
 
diff --git a/lib/mastodon/emoji_cli.rb b/lib/mastodon/emoji_cli.rb
index 97a822e45..beac1b1fd 100644
--- a/lib/mastodon/emoji_cli.rb
+++ b/lib/mastodon/emoji_cli.rb
@@ -15,6 +15,7 @@ module Mastodon
     option :suffix
     option :overwrite, type: :boolean
     option :unlisted, type: :boolean
+    option :category
     desc 'import PATH', 'Import emoji from a TAR GZIP archive at PATH'
     long_desc <<-LONG_DESC
       Imports custom emoji from a TAR GZIP archive specified by PATH.
@@ -22,6 +23,9 @@ module Mastodon
       Existing emoji will be skipped unless the --overwrite option
       is provided, in which case they will be overwritten.
 
+      You can specifiy a --category under which the emojis will be
+      grouped together.
+
       With the --prefix option, a prefix can be added to all
       generated shortcodes. Likewise, the --suffix option controls
       the suffix of all shortcodes.
@@ -33,6 +37,7 @@ module Mastodon
       imported = 0
       skipped  = 0
       failed   = 0
+      category = options[:category] ? CustomEmojiCategory.find_or_create_by(name: options[:category]) : nil
 
       Gem::Package::TarReader.new(Zlib::GzipReader.open(path)) do |tar|
         tar.each do |entry|
@@ -50,6 +55,7 @@ module Mastodon
           custom_emoji.image = StringIO.new(entry.read)
           custom_emoji.image_file_name = File.basename(entry.full_name)
           custom_emoji.visible_in_picker = !options[:unlisted]
+          custom_emoji.category = category
 
           if custom_emoji.save
             imported += 1
diff --git a/spec/fabricators/custom_emoji_category_fabricator.rb b/spec/fabricators/custom_emoji_category_fabricator.rb
new file mode 100644
index 000000000..f593b95ed
--- /dev/null
+++ b/spec/fabricators/custom_emoji_category_fabricator.rb
@@ -0,0 +1,3 @@
+Fabricator(:custom_emoji_category) do
+  name "MyString"
+end
diff --git a/spec/models/custom_emoji_category_spec.rb b/spec/models/custom_emoji_category_spec.rb
new file mode 100644
index 000000000..160033f4d
--- /dev/null
+++ b/spec/models/custom_emoji_category_spec.rb
@@ -0,0 +1,5 @@
+require 'rails_helper'
+
+RSpec.describe CustomEmojiCategory, type: :model do
+  pending "add some examples to (or delete) #{__FILE__}"
+end
diff --git a/yarn.lock b/yarn.lock
index 560d84e31..2c402782c 100644
--- a/yarn.lock
+++ b/yarn.lock
@@ -3384,8 +3384,8 @@ elliptic@^6.0.0:
     minimalistic-crypto-utils "^1.0.0"
 
 emoji-mart@Gargron/emoji-mart#build:
-  version "2.6.2"
-  resolved "https://codeload.github.com/Gargron/emoji-mart/tar.gz/ff00dc470b5b2d9f145a6d6e977a54de5df2b4c9"
+  version "2.6.3"
+  resolved "https://codeload.github.com/Gargron/emoji-mart/tar.gz/934f314fd8322276765066e8a2a6be5bac61b1cf"
 
 emoji-regex@^7.0.1, emoji-regex@^7.0.2:
   version "7.0.3"
-- 
cgit 


From 0d9ffe56fb59e0d1fce91265f44140d874c0bfba Mon Sep 17 00:00:00 2001
From: Eugen Rochko <eugen@zeonfederated.com>
Date: Tue, 2 Jul 2019 00:34:38 +0200
Subject: Add request pool to improve delivery performance (#10353)

* Add request pool to improve delivery performance

Fix #7909

* Ensure connection is closed when exception interrupts execution

* Remove Timeout#timeout from socket connection

* Fix infinite retrial loop on HTTP::ConnectionError

* Close sockets on failure, reduce idle time to 90 seconds

* Add MAX_REQUEST_POOL_SIZE option to limit concurrent connections to the same server

* Use a shared pool size, 512 by default, to stay below open file limit

* Add some tests

* Add more tests

* Reduce MAX_IDLE_TIME from 90 to 30 seconds, reap every 30 seconds

* Use a shared pool that returns preferred connection but re-purposes other ones when needed

* Fix wrong connection being returned on subsequent calls within the same thread

* Reduce mutex calls on flushes from 2 to 1 and add test for reaping
---
 Gemfile                                            |   1 +
 Gemfile.lock                                       |   1 +
 app/lib/connection_pool/shared_connection_pool.rb  |  63 ++++++++++++
 app/lib/connection_pool/shared_timed_stack.rb      |  95 +++++++++++++++++
 app/lib/request.rb                                 |  70 +++++++++----
 app/lib/request_pool.rb                            | 114 +++++++++++++++++++++
 app/workers/activitypub/delivery_worker.rb         |  15 ++-
 .../connection_pool/shared_connection_pool_spec.rb |  28 +++++
 .../lib/connection_pool/shared_timed_stack_spec.rb |  61 +++++++++++
 spec/lib/request_pool_spec.rb                      |  63 ++++++++++++
 10 files changed, 488 insertions(+), 23 deletions(-)
 create mode 100644 app/lib/connection_pool/shared_connection_pool.rb
 create mode 100644 app/lib/connection_pool/shared_timed_stack.rb
 create mode 100644 app/lib/request_pool.rb
 create mode 100644 spec/lib/connection_pool/shared_connection_pool_spec.rb
 create mode 100644 spec/lib/connection_pool/shared_timed_stack_spec.rb
 create mode 100644 spec/lib/request_pool_spec.rb

(limited to 'spec')

diff --git a/Gemfile b/Gemfile
index e99a808d4..2fcb603c3 100644
--- a/Gemfile
+++ b/Gemfile
@@ -148,3 +148,4 @@ group :production do
 end
 
 gem 'concurrent-ruby', require: false
+gem 'connection_pool', require: false
diff --git a/Gemfile.lock b/Gemfile.lock
index c8856ccc5..6043a2fa9 100644
--- a/Gemfile.lock
+++ b/Gemfile.lock
@@ -666,6 +666,7 @@ DEPENDENCIES
   cld3 (~> 3.2.4)
   climate_control (~> 0.2)
   concurrent-ruby
+  connection_pool
   derailed_benchmarks
   devise (~> 4.6)
   devise-two-factor (~> 3.0)
diff --git a/app/lib/connection_pool/shared_connection_pool.rb b/app/lib/connection_pool/shared_connection_pool.rb
new file mode 100644
index 000000000..2865a4108
--- /dev/null
+++ b/app/lib/connection_pool/shared_connection_pool.rb
@@ -0,0 +1,63 @@
+# frozen_string_literal: true
+
+require 'connection_pool'
+require_relative './shared_timed_stack'
+
+class ConnectionPool::SharedConnectionPool < ConnectionPool
+  def initialize(options = {}, &block)
+    super(options, &block)
+
+    @available = ConnectionPool::SharedTimedStack.new(@size, &block)
+  end
+
+  delegate :size, :flush, to: :@available
+
+  def with(preferred_tag, options = {})
+    Thread.handle_interrupt(Exception => :never) do
+      conn = checkout(preferred_tag, options)
+
+      begin
+        Thread.handle_interrupt(Exception => :immediate) do
+          yield conn
+        end
+      ensure
+        checkin(preferred_tag)
+      end
+    end
+  end
+
+  def checkout(preferred_tag, options = {})
+    if ::Thread.current[key(preferred_tag)]
+      ::Thread.current[key_count(preferred_tag)] += 1
+      ::Thread.current[key(preferred_tag)]
+    else
+      ::Thread.current[key_count(preferred_tag)] = 1
+      ::Thread.current[key(preferred_tag)] = @available.pop(preferred_tag, options[:timeout] || @timeout)
+    end
+  end
+
+  def checkin(preferred_tag)
+    if ::Thread.current[key(preferred_tag)]
+      if ::Thread.current[key_count(preferred_tag)] == 1
+        @available.push(::Thread.current[key(preferred_tag)])
+        ::Thread.current[key(preferred_tag)] = nil
+      else
+        ::Thread.current[key_count(preferred_tag)] -= 1
+      end
+    else
+      raise ConnectionPool::Error, 'no connections are checked out'
+    end
+
+    nil
+  end
+
+  private
+
+  def key(tag)
+    :"#{@key}-#{tag}"
+  end
+
+  def key_count(tag)
+    :"#{@key_count}-#{tag}"
+  end
+end
diff --git a/app/lib/connection_pool/shared_timed_stack.rb b/app/lib/connection_pool/shared_timed_stack.rb
new file mode 100644
index 000000000..14a5285c4
--- /dev/null
+++ b/app/lib/connection_pool/shared_timed_stack.rb
@@ -0,0 +1,95 @@
+# frozen_string_literal: true
+
+class ConnectionPool::SharedTimedStack
+  def initialize(max = 0, &block)
+    @create_block = block
+    @max          = max
+    @created      = 0
+    @queue        = []
+    @tagged_queue = Hash.new { |hash, key| hash[key] = [] }
+    @mutex        = Mutex.new
+    @resource     = ConditionVariable.new
+  end
+
+  def push(connection)
+    @mutex.synchronize do
+      store_connection(connection)
+      @resource.broadcast
+    end
+  end
+
+  alias << push
+
+  def pop(preferred_tag, timeout = 5.0)
+    deadline = current_time + timeout
+
+    @mutex.synchronize do
+      loop do
+        return fetch_preferred_connection(preferred_tag) unless @tagged_queue[preferred_tag].empty?
+
+        connection = try_create(preferred_tag)
+        return connection if connection
+
+        to_wait = deadline - current_time
+        raise Timeout::Error, "Waited #{timeout} sec" if to_wait <= 0
+
+        @resource.wait(@mutex, to_wait)
+      end
+    end
+  end
+
+  def empty?
+    size.zero?
+  end
+
+  def size
+    @mutex.synchronize do
+      @queue.size
+    end
+  end
+
+  def flush
+    @mutex.synchronize do
+      @queue.delete_if do |connection|
+        delete = !connection.in_use && (connection.dead || connection.seconds_idle >= RequestPool::MAX_IDLE_TIME)
+
+        if delete
+          @tagged_queue[connection.site].delete(connection)
+          connection.close
+          @created -= 1
+        end
+
+        delete
+      end
+    end
+  end
+
+  private
+
+  def try_create(preferred_tag)
+    if @created == @max && !@queue.empty?
+      throw_away_connection = @queue.pop
+      @tagged_queue[throw_away_connection.site].delete(throw_away_connection)
+      @create_block.call(preferred_tag)
+    elsif @created != @max
+      connection = @create_block.call(preferred_tag)
+      @created += 1
+      connection
+    end
+  end
+
+  def fetch_preferred_connection(preferred_tag)
+    connection = @tagged_queue[preferred_tag].pop
+    @queue.delete(connection)
+    connection
+  end
+
+  def current_time
+    Process.clock_gettime(Process::CLOCK_MONOTONIC)
+  end
+
+  def store_connection(connection)
+    @tagged_queue[connection.site].push(connection)
+    @queue.push(connection)
+  end
+end
diff --git a/app/lib/request.rb b/app/lib/request.rb
index e555ae6a1..af49d6c77 100644
--- a/app/lib/request.rb
+++ b/app/lib/request.rb
@@ -17,15 +17,21 @@ end
 class Request
   REQUEST_TARGET = '(request-target)'
 
+  # We enforce a 5s timeout on DNS resolving, 5s timeout on socket opening
+  # and 5s timeout on the TLS handshake, meaning the worst case should take
+  # about 15s in total
+  TIMEOUT = { connect: 5, read: 10, write: 10 }.freeze
+
   include RoutingHelper
 
   def initialize(verb, url, **options)
     raise ArgumentError if url.blank?
 
-    @verb    = verb
-    @url     = Addressable::URI.parse(url).normalize
-    @options = options.merge(use_proxy? ? Rails.configuration.x.http_client_proxy : { socket_class: Socket })
-    @headers = {}
+    @verb        = verb
+    @url         = Addressable::URI.parse(url).normalize
+    @http_client = options.delete(:http_client)
+    @options     = options.merge(use_proxy? ? Rails.configuration.x.http_client_proxy : { socket_class: Socket })
+    @headers     = {}
 
     raise Mastodon::HostValidationError, 'Instance does not support hidden service connections' if block_hidden_service?
 
@@ -50,15 +56,24 @@ class Request
 
   def perform
     begin
-      response = http_client.headers(headers).public_send(@verb, @url.to_s, @options)
+      response = http_client.public_send(@verb, @url.to_s, @options.merge(headers: headers))
     rescue => e
       raise e.class, "#{e.message} on #{@url}", e.backtrace[0]
     end
 
     begin
-      yield response.extend(ClientLimit) if block_given?
+      response = response.extend(ClientLimit)
+
+      # If we are using a persistent connection, we have to
+      # read every response to be able to move forward at all.
+      # However, simply calling #to_s or #flush may not be safe,
+      # as the response body, if malicious, could be too big
+      # for our memory. So we use the #body_with_limit method
+      response.body_with_limit if http_client.persistent?
+
+      yield response if block_given?
     ensure
-      http_client.close
+      http_client.close unless http_client.persistent?
     end
   end
 
@@ -76,6 +91,10 @@ class Request
 
       %w(http https).include?(parsed_url.scheme) && parsed_url.host.present?
     end
+
+    def http_client
+      HTTP.use(:auto_inflate).timeout(:per_operation, TIMEOUT.dup).follow(max_hops: 2)
+    end
   end
 
   private
@@ -116,16 +135,8 @@ class Request
     end
   end
 
-  def timeout
-    # We enforce a 1s timeout on DNS resolving, 10s timeout on socket opening
-    # and 5s timeout on the TLS handshake, meaning the worst case should take
-    # about 16s in total
-
-    { connect: 5, read: 10, write: 10 }
-  end
-
   def http_client
-    @http_client ||= HTTP.use(:auto_inflate).timeout(:per_operation, timeout).follow(max_hops: 2)
+    @http_client ||= Request.http_client
   end
 
   def use_proxy?
@@ -169,20 +180,41 @@ class Request
         return super(host, *args) if thru_hidden_service?(host)
 
         outer_e = nil
+        port    = args.first
 
         Resolv::DNS.open do |dns|
           dns.timeouts = 5
 
           addresses = dns.getaddresses(host).take(2)
-          time_slot = 10.0 / addresses.size
 
           addresses.each do |address|
             begin
               raise Mastodon::HostValidationError if PrivateAddressCheck.private_address?(IPAddr.new(address.to_s))
 
-              ::Timeout.timeout(time_slot, HTTP::TimeoutError) do
-                return super(address.to_s, *args)
+              sock     = ::Socket.new(::Socket::AF_INET, ::Socket::SOCK_STREAM, 0)
+              sockaddr = ::Socket.pack_sockaddr_in(port, address.to_s)
+
+              sock.setsockopt(::Socket::IPPROTO_TCP, ::Socket::TCP_NODELAY, 1)
+
+              begin
+                sock.connect_nonblock(sockaddr)
+              rescue IO::WaitWritable
+                if IO.select(nil, [sock], nil, Request::TIMEOUT[:connect])
+                  begin
+                    sock.connect_nonblock(sockaddr)
+                  rescue Errno::EISCONN
+                    # Yippee!
+                  rescue
+                    sock.close
+                    raise
+                  end
+                else
+                  sock.close
+                  raise HTTP::TimeoutError, "Connect timed out after #{Request::TIMEOUT[:connect]} seconds"
+                end
               end
+
+              return sock
             rescue => e
               outer_e = e
             end
diff --git a/app/lib/request_pool.rb b/app/lib/request_pool.rb
new file mode 100644
index 000000000..e5899a79a
--- /dev/null
+++ b/app/lib/request_pool.rb
@@ -0,0 +1,114 @@
+# frozen_string_literal: true
+
+require_relative './connection_pool/shared_connection_pool'
+
+class RequestPool
+  def self.current
+    @current ||= RequestPool.new
+  end
+
+  class Reaper
+    attr_reader :pool, :frequency
+
+    def initialize(pool, frequency)
+      @pool      = pool
+      @frequency = frequency
+    end
+
+    def run
+      return unless frequency&.positive?
+
+      Thread.new(frequency, pool) do |t, p|
+        loop do
+          sleep t
+          p.flush
+        end
+      end
+    end
+  end
+
+  MAX_IDLE_TIME = 30
+  WAIT_TIMEOUT  = 5
+  MAX_POOL_SIZE = ENV.fetch('MAX_REQUEST_POOL_SIZE', 512).to_i
+
+  class Connection
+    attr_reader :site, :last_used_at, :created_at, :in_use, :dead, :fresh
+
+    def initialize(site)
+      @site         = site
+      @http_client  = http_client
+      @last_used_at = nil
+      @created_at   = current_time
+      @dead         = false
+      @fresh        = true
+    end
+
+    def use
+      @last_used_at = current_time
+      @in_use       = true
+
+      retries = 0
+
+      begin
+        yield @http_client
+      rescue HTTP::ConnectionError
+        # It's possible the connection was closed, so let's
+        # try re-opening it once
+
+        close
+
+        if @fresh || retries.positive?
+          raise
+        else
+          @http_client = http_client
+          retries     += 1
+          retry
+        end
+      rescue StandardError
+        # If this connection raises errors of any kind, it's
+        # better if it gets reaped as soon as possible
+
+        close
+        @dead = true
+        raise
+      end
+    ensure
+      @fresh  = false
+      @in_use = false
+    end
+
+    def seconds_idle
+      current_time - (@last_used_at || @created_at)
+    end
+
+    def close
+      @http_client.close
+    end
+
+    private
+
+    def http_client
+      Request.http_client.persistent(@site, timeout: MAX_IDLE_TIME)
+    end
+
+    def current_time
+      Process.clock_gettime(Process::CLOCK_MONOTONIC)
+    end
+  end
+
+  def initialize
+    @pool   = ConnectionPool::SharedConnectionPool.new(size: MAX_POOL_SIZE, timeout: WAIT_TIMEOUT) { |site| Connection.new(site) }
+    @reaper = Reaper.new(self, 30)
+    @reaper.run
+  end
+
+  def with(site, &block)
+    @pool.with(site) do |connection|
+      ActiveSupport::Notifications.instrument('with.request_pool', miss: connection.fresh, host: connection.site) do
+        connection.use(&block)
+      end
+    end
+  end
+
+  delegate :size, :flush, to: :@pool
+end
diff --git a/app/workers/activitypub/delivery_worker.rb b/app/workers/activitypub/delivery_worker.rb
index 5e4c391f0..79f1e8153 100644
--- a/app/workers/activitypub/delivery_worker.rb
+++ b/app/workers/activitypub/delivery_worker.rb
@@ -17,6 +17,7 @@ class ActivityPub::DeliveryWorker
     @json           = json
     @source_account = Account.find(source_account_id)
     @inbox_url      = inbox_url
+    @host           = Addressable::URI.parse(inbox_url).normalized_site
 
     perform_request
 
@@ -28,16 +29,18 @@ class ActivityPub::DeliveryWorker
 
   private
 
-  def build_request
-    request = Request.new(:post, @inbox_url, body: @json)
+  def build_request(http_client)
+    request = Request.new(:post, @inbox_url, body: @json, http_client: http_client)
     request.on_behalf_of(@source_account, :uri, sign_with: @options[:sign_with])
     request.add_headers(HEADERS)
   end
 
   def perform_request
     light = Stoplight(@inbox_url) do
-      build_request.perform do |response|
-        raise Mastodon::UnexpectedResponseError, response unless response_successful?(response) || response_error_unsalvageable?(response)
+      request_pool.with(@host) do |http_client|
+        build_request(http_client).perform do |response|
+          raise Mastodon::UnexpectedResponseError, response unless response_successful?(response) || response_error_unsalvageable?(response)
+        end
       end
     end
 
@@ -57,4 +60,8 @@ class ActivityPub::DeliveryWorker
   def failure_tracker
     @failure_tracker ||= DeliveryFailureTracker.new(@inbox_url)
   end
+
+  def request_pool
+    RequestPool.current
+  end
 end
diff --git a/spec/lib/connection_pool/shared_connection_pool_spec.rb b/spec/lib/connection_pool/shared_connection_pool_spec.rb
new file mode 100644
index 000000000..114464558
--- /dev/null
+++ b/spec/lib/connection_pool/shared_connection_pool_spec.rb
@@ -0,0 +1,28 @@
+# frozen_string_literal: true
+
+require 'rails_helper'
+
+describe ConnectionPool::SharedConnectionPool do
+  class MiniConnection
+    attr_reader :site
+
+    def initialize(site)
+      @site = site
+    end
+  end
+
+  subject { described_class.new(size: 5, timeout: 5) { |site| MiniConnection.new(site) } }
+
+  describe '#with' do
+    it 'runs a block with a connection' do
+      block_run = false
+
+      subject.with('foo') do |connection|
+        expect(connection).to be_a MiniConnection
+        block_run = true
+      end
+
+      expect(block_run).to be true
+    end
+  end
+end
diff --git a/spec/lib/connection_pool/shared_timed_stack_spec.rb b/spec/lib/connection_pool/shared_timed_stack_spec.rb
new file mode 100644
index 000000000..f680c5966
--- /dev/null
+++ b/spec/lib/connection_pool/shared_timed_stack_spec.rb
@@ -0,0 +1,61 @@
+# frozen_string_literal: true
+
+require 'rails_helper'
+
+describe ConnectionPool::SharedTimedStack do
+  class MiniConnection
+    attr_reader :site
+
+    def initialize(site)
+      @site = site
+    end
+  end
+
+  subject { described_class.new(5) { |site| MiniConnection.new(site) } }
+
+  describe '#push' do
+    it 'keeps the connection in the stack' do
+      subject.push(MiniConnection.new('foo'))
+      expect(subject.size).to eq 1
+    end
+  end
+
+  describe '#pop' do
+    it 'returns a connection' do
+      expect(subject.pop('foo')).to be_a MiniConnection
+    end
+
+    it 'returns the same connection that was pushed in' do
+      connection = MiniConnection.new('foo')
+      subject.push(connection)
+      expect(subject.pop('foo')).to be connection
+    end
+
+    it 'does not create more than maximum amount of connections' do
+      expect { 6.times { subject.pop('foo', 0) } }.to raise_error Timeout::Error
+    end
+
+    it 'repurposes a connection for a different site when maximum amount is reached' do
+      5.times { subject.push(MiniConnection.new('foo')) }
+      expect(subject.pop('bar')).to be_a MiniConnection
+    end
+  end
+
+  describe '#empty?' do
+    it 'returns true when no connections on the stack' do
+      expect(subject.empty?).to be true
+    end
+
+    it 'returns false when there are connections on the stack' do
+      subject.push(MiniConnection.new('foo'))
+      expect(subject.empty?).to be false
+    end
+  end
+
+  describe '#size' do
+    it 'returns the number of connections on the stack' do
+      2.times { subject.push(MiniConnection.new('foo')) }
+      expect(subject.size).to eq 2
+    end
+  end
+end
diff --git a/spec/lib/request_pool_spec.rb b/spec/lib/request_pool_spec.rb
new file mode 100644
index 000000000..4a144d7c7
--- /dev/null
+++ b/spec/lib/request_pool_spec.rb
@@ -0,0 +1,63 @@
+# frozen_string_literal: true
+
+require 'rails_helper'
+
+describe RequestPool do
+  subject { described_class.new }
+
+  describe '#with' do
+    it 'returns a HTTP client for a host' do
+      subject.with('http://example.com') do |http_client|
+        expect(http_client).to be_a HTTP::Client
+      end
+    end
+
+    it 'returns the same instance of HTTP client within the same thread for the same host' do
+      test_client = nil
+
+      subject.with('http://example.com') { |http_client| test_client = http_client }
+      expect(test_client).to_not be_nil
+      subject.with('http://example.com') { |http_client| expect(http_client).to be test_client }
+    end
+
+    it 'returns different HTTP clients for different hosts' do
+      test_client = nil
+
+      subject.with('http://example.com') { |http_client| test_client = http_client }
+      expect(test_client).to_not be_nil
+      subject.with('http://example.org') { |http_client| expect(http_client).to_not be test_client }
+    end
+
+    it 'grows to the number of threads accessing it' do
+      stub_request(:get, 'http://example.com/').to_return(status: 200, body: 'Hello!')
+
+      subject
+
+      threads = 20.times.map do |i|
+        Thread.new do
+          20.times do
+            subject.with('http://example.com') do |http_client|
+              http_client.get('/').flush
+            end
+          end
+        end
+      end
+
+      threads.map(&:join)
+
+      expect(subject.size).to be > 1
+    end
+
+    it 'closes idle connections' do
+      stub_request(:get, 'http://example.com/').to_return(status: 200, body: 'Hello!')
+
+      subject.with('http://example.com') do |http_client|
+        http_client.get('/').flush
+      end
+
+      expect(subject.size).to eq 1
+      sleep RequestPool::MAX_IDLE_TIME + 30 + 1
+      expect(subject.size).to eq 0
+    end
+  end
+end
-- 
cgit 


From 3fd6ab99e69e736e10c7022f07dfa34b60a4e54d Mon Sep 17 00:00:00 2001
From: Eugen Rochko <eugen@zeonfederated.com>
Date: Fri, 5 Jul 2019 02:14:56 +0200
Subject: Remove deprecated REST API `GET /api/v1/timelines/direct` (#11212)

---
 .../api/v1/timelines/direct_controller.rb          | 63 ----------------------
 app/models/status.rb                               | 41 --------------
 config/routes.rb                                   |  1 -
 .../api/v1/timelines/direct_controller_spec.rb     | 17 ------
 spec/models/status_spec.rb                         | 50 -----------------
 5 files changed, 172 deletions(-)
 delete mode 100644 app/controllers/api/v1/timelines/direct_controller.rb
 delete mode 100644 spec/controllers/api/v1/timelines/direct_controller_spec.rb

(limited to 'spec')

diff --git a/app/controllers/api/v1/timelines/direct_controller.rb b/app/controllers/api/v1/timelines/direct_controller.rb
deleted file mode 100644
index d8a76d153..000000000
--- a/app/controllers/api/v1/timelines/direct_controller.rb
+++ /dev/null
@@ -1,63 +0,0 @@
-# frozen_string_literal: true
-
-class Api::V1::Timelines::DirectController < Api::BaseController
-  before_action -> { doorkeeper_authorize! :read, :'read:statuses' }, only: [:show]
-  before_action :require_user!, only: [:show]
-  after_action :insert_pagination_headers, unless: -> { @statuses.empty? }
-
-  respond_to :json
-
-  def show
-    @statuses = load_statuses
-    render json: @statuses, each_serializer: REST::StatusSerializer, relationships: StatusRelationshipsPresenter.new(@statuses, current_user&.account_id)
-  end
-
-  private
-
-  def load_statuses
-    cached_direct_statuses
-  end
-
-  def cached_direct_statuses
-    cache_collection direct_statuses, Status
-  end
-
-  def direct_statuses
-    direct_timeline_statuses
-  end
-
-  def direct_timeline_statuses
-    # this query requires built in pagination.
-    Status.as_direct_timeline(
-      current_account,
-      limit_param(DEFAULT_STATUSES_LIMIT),
-      params[:max_id],
-      params[:since_id],
-      true # returns array of cache_ids object
-    )
-  end
-
-  def insert_pagination_headers
-    set_pagination_headers(next_path, prev_path)
-  end
-
-  def pagination_params(core_params)
-    params.permit(:local, :limit).merge(core_params)
-  end
-
-  def next_path
-    api_v1_timelines_direct_url pagination_params(max_id: pagination_max_id)
-  end
-
-  def prev_path
-    api_v1_timelines_direct_url pagination_params(since_id: pagination_since_id)
-  end
-
-  def pagination_max_id
-    @statuses.last.id
-  end
-
-  def pagination_since_id
-    @statuses.first.id
-  end
-end
diff --git a/app/models/status.rb b/app/models/status.rb
index fb9bbc9a9..2258e2d07 100644
--- a/app/models/status.rb
+++ b/app/models/status.rb
@@ -280,47 +280,6 @@ class Status < ApplicationRecord
       where(account: [account] + account.following).where(visibility: [:public, :unlisted, :private])
     end
 
-    def as_direct_timeline(account, limit = 20, max_id = nil, since_id = nil, cache_ids = false)
-      # direct timeline is mix of direct message from_me and to_me.
-      # 2 queries are executed with pagination.
-      # constant expression using arel_table is required for partial index
-
-      # _from_me part does not require any timeline filters
-      query_from_me = where(account_id: account.id)
-                      .where(Status.arel_table[:visibility].eq(3))
-                      .limit(limit)
-                      .order('statuses.id DESC')
-
-      # _to_me part requires mute and block filter.
-      # FIXME: may we check mutes.hide_notifications?
-      query_to_me = Status
-                    .joins(:mentions)
-                    .merge(Mention.where(account_id: account.id))
-                    .where(Status.arel_table[:visibility].eq(3))
-                    .limit(limit)
-                    .order('mentions.status_id DESC')
-                    .not_excluded_by_account(account)
-
-      if max_id.present?
-        query_from_me = query_from_me.where('statuses.id < ?', max_id)
-        query_to_me = query_to_me.where('mentions.status_id < ?', max_id)
-      end
-
-      if since_id.present?
-        query_from_me = query_from_me.where('statuses.id > ?', since_id)
-        query_to_me = query_to_me.where('mentions.status_id > ?', since_id)
-      end
-
-      if cache_ids
-        # returns array of cache_ids object that have id and updated_at
-        (query_from_me.cache_ids.to_a + query_to_me.cache_ids.to_a).uniq(&:id).sort_by(&:id).reverse.take(limit)
-      else
-        # returns ActiveRecord.Relation
-        items = (query_from_me.select(:id).to_a + query_to_me.select(:id).to_a).uniq(&:id).sort_by(&:id).reverse.take(limit)
-        Status.where(id: items.map(&:id))
-      end
-    end
-
     def as_public_timeline(account = nil, local_only = false)
       query = timeline_scope(local_only).without_replies
 
diff --git a/config/routes.rb b/config/routes.rb
index 764db8db2..29db54a32 100644
--- a/config/routes.rb
+++ b/config/routes.rb
@@ -299,7 +299,6 @@ Rails.application.routes.draw do
       end
 
       namespace :timelines do
-        resource :direct, only: :show, controller: :direct
         resource :home, only: :show, controller: :home
         resource :public, only: :show, controller: :public
         resources :tag, only: :show
diff --git a/spec/controllers/api/v1/timelines/direct_controller_spec.rb b/spec/controllers/api/v1/timelines/direct_controller_spec.rb
deleted file mode 100644
index a22c2cbea..000000000
--- a/spec/controllers/api/v1/timelines/direct_controller_spec.rb
+++ /dev/null
@@ -1,17 +0,0 @@
-# frozen_string_literal: true
-
-require 'rails_helper'
-
-RSpec.describe Api::V1::Timelines::DirectController, type: :controller do
-  let(:user)  { Fabricate(:user) }
-  let(:token) { Fabricate(:accessible_access_token, resource_owner_id: user.id, scopes: 'read:statuses') }
-
-  describe 'GET #show' do
-    it 'returns 200' do
-      allow(controller).to receive(:doorkeeper_token) { token }
-      get :show
-
-      expect(response).to have_http_status(200)
-    end
-  end
-end
diff --git a/spec/models/status_spec.rb b/spec/models/status_spec.rb
index e8cf18af9..a8c567414 100644
--- a/spec/models/status_spec.rb
+++ b/spec/models/status_spec.rb
@@ -339,56 +339,6 @@ RSpec.describe Status, type: :model do
     end
   end
 
-  describe '.as_direct_timeline' do
-    let(:account) { Fabricate(:account) }
-    let(:followed) { Fabricate(:account) }
-    let(:not_followed) { Fabricate(:account) }
-
-    before do
-      Fabricate(:follow, account: account, target_account: followed)
-
-      @self_public_status = Fabricate(:status, account: account, visibility: :public)
-      @self_direct_status = Fabricate(:status, account: account, visibility: :direct)
-      @followed_public_status = Fabricate(:status, account: followed, visibility: :public)
-      @followed_direct_status = Fabricate(:status, account: followed, visibility: :direct)
-      @not_followed_direct_status = Fabricate(:status, account: not_followed, visibility: :direct)
-
-      @results = Status.as_direct_timeline(account)
-    end
-
-    it 'does not include public statuses from self' do
-      expect(@results).to_not include(@self_public_status)
-    end
-
-    it 'includes direct statuses from self' do
-      expect(@results).to include(@self_direct_status)
-    end
-
-    it 'does not include public statuses from followed' do
-      expect(@results).to_not include(@followed_public_status)
-    end
-
-    it 'does not include direct statuses not mentioning recipient from followed' do
-      expect(@results).to_not include(@followed_direct_status)
-    end
-
-    it 'does not include direct statuses not mentioning recipient from non-followed' do
-      expect(@results).to_not include(@not_followed_direct_status)
-    end
-
-    it 'includes direct statuses mentioning recipient from followed' do
-      Fabricate(:mention, account: account, status: @followed_direct_status)
-      results2 = Status.as_direct_timeline(account)
-      expect(results2).to include(@followed_direct_status)
-    end
-
-    it 'includes direct statuses mentioning recipient from non-followed' do
-      Fabricate(:mention, account: account, status: @not_followed_direct_status)
-      results2 = Status.as_direct_timeline(account)
-      expect(results2).to include(@not_followed_direct_status)
-    end
-  end
-
   describe '.as_public_timeline' do
     it 'only includes statuses with public visibility' do
       public_status = Fabricate(:status, visibility: :public)
-- 
cgit 


From 0c1b1069c96d3452d43d922a737c522d10e4747b Mon Sep 17 00:00:00 2001
From: Eugen Rochko <eugen@zeonfederated.com>
Date: Fri, 5 Jul 2019 02:15:24 +0200
Subject: Remove deprecated REST API `GET /api/v1/statuses/:id/card` (#11213)

---
 app/controllers/api/v1/statuses_controller.rb       | 14 ++------------
 config/routes.rb                                    |  1 -
 spec/controllers/api/v1/statuses_controller_spec.rb | 14 --------------
 3 files changed, 2 insertions(+), 27 deletions(-)

(limited to 'spec')

diff --git a/app/controllers/api/v1/statuses_controller.rb b/app/controllers/api/v1/statuses_controller.rb
index b0e134554..71a505c26 100644
--- a/app/controllers/api/v1/statuses_controller.rb
+++ b/app/controllers/api/v1/statuses_controller.rb
@@ -5,8 +5,8 @@ class Api::V1::StatusesController < Api::BaseController
 
   before_action -> { authorize_if_got_token! :read, :'read:statuses' }, except: [:create, :destroy]
   before_action -> { doorkeeper_authorize! :write, :'write:statuses' }, only:   [:create, :destroy]
-  before_action :require_user!, except:  [:show, :context, :card]
-  before_action :set_status, only:       [:show, :context, :card]
+  before_action :require_user!, except:  [:show, :context]
+  before_action :set_status, only:       [:show, :context]
 
   respond_to :json
 
@@ -33,16 +33,6 @@ class Api::V1::StatusesController < Api::BaseController
     render json: @context, serializer: REST::ContextSerializer, relationships: StatusRelationshipsPresenter.new(statuses, current_user&.account_id)
   end
 
-  def card
-    @card = @status.preview_cards.first
-
-    if @card.nil?
-      render_empty
-    else
-      render json: @card, serializer: REST::PreviewCardSerializer
-    end
-  end
-
   def create
     @status = PostStatusService.new.call(current_user.account,
                                          text: status_params[:status],
diff --git a/config/routes.rb b/config/routes.rb
index 1f97dea4c..9ab5ba7f0 100644
--- a/config/routes.rb
+++ b/config/routes.rb
@@ -294,7 +294,6 @@ Rails.application.routes.draw do
 
         member do
           get :context
-          get :card
         end
       end
 
diff --git a/spec/controllers/api/v1/statuses_controller_spec.rb b/spec/controllers/api/v1/statuses_controller_spec.rb
index 8bc3b0c67..9ff5fcd3b 100644
--- a/spec/controllers/api/v1/statuses_controller_spec.rb
+++ b/spec/controllers/api/v1/statuses_controller_spec.rb
@@ -91,13 +91,6 @@ RSpec.describe Api::V1::StatusesController, type: :controller do
           expect(response).to have_http_status(404)
         end
       end
-
-      describe 'GET #card' do
-        it 'returns http unautharized' do
-          get :card, params: { id: status.id }
-          expect(response).to have_http_status(404)
-        end
-      end
     end
 
     context 'with a public status' do
@@ -120,13 +113,6 @@ RSpec.describe Api::V1::StatusesController, type: :controller do
           expect(response).to have_http_status(200)
         end
       end
-
-      describe 'GET #card' do
-        it 'returns http success' do
-          get :card, params: { id: status.id }
-          expect(response).to have_http_status(200)
-        end
-      end
     end
   end
 end
-- 
cgit 


From 23aeef52cc4540b4514e9f3b935b21f0530a3746 Mon Sep 17 00:00:00 2001
From: Eugen Rochko <eugen@zeonfederated.com>
Date: Sat, 6 Jul 2019 23:26:16 +0200
Subject: Remove Salmon and PubSubHubbub (#11205)

* Remove Salmon and PubSubHubbub endpoints

* Add error when trying to follow OStatus accounts

* Fix new accounts not being created in ResolveAccountService
---
 app/controllers/activitypub/inboxes_controller.rb  |   1 -
 app/controllers/admin/accounts_controller.rb       |  16 +-
 app/controllers/api/push_controller.rb             |  73 ------
 app/controllers/api/salmon_controller.rb           |  37 ---
 app/controllers/api/subscriptions_controller.rb    |  51 -----
 app/controllers/api/v1/follows_controller.rb       |  31 ---
 app/lib/ostatus/activity/base.rb                   |  71 ------
 app/lib/ostatus/activity/creation.rb               | 219 ------------------
 app/lib/ostatus/activity/deletion.rb               |  16 --
 app/lib/ostatus/activity/general.rb                |  20 --
 app/lib/ostatus/activity/post.rb                   |  23 --
 app/lib/ostatus/activity/remote.rb                 |  11 -
 app/lib/ostatus/activity/share.rb                  |  26 ---
 app/lib/ostatus/atom_serializer.rb                 |   2 -
 app/models/account.rb                              |   3 +-
 app/serializers/webfinger_serializer.rb            |   1 -
 app/services/authorize_follow_service.rb           |  12 +-
 app/services/batched_remove_status_service.rb      |  34 +--
 app/services/block_domain_service.rb               |   1 -
 app/services/block_service.rb                      |  12 +-
 app/services/concerns/author_extractor.rb          |  23 --
 app/services/concerns/stream_entry_renderer.rb     |   7 -
 app/services/favourite_service.rb                  |   6 -
 app/services/fetch_remote_account_service.rb       |  28 ---
 app/services/fetch_remote_status_service.rb        |  28 ---
 app/services/follow_service.rb                     |  24 +-
 app/services/post_status_service.rb                |   1 -
 app/services/process_feed_service.rb               |  31 ---
 app/services/process_interaction_service.rb        | 151 ------------
 app/services/process_mentions_service.rb           |   7 -
 app/services/pubsubhubbub/subscribe_service.rb     |  53 -----
 app/services/pubsubhubbub/unsubscribe_service.rb   |  31 ---
 app/services/reblog_service.rb                     |   4 -
 app/services/reject_follow_service.rb              |  12 +-
 app/services/remove_status_service.rb              |  13 --
 app/services/resolve_account_service.rb            | 131 +----------
 app/services/send_interaction_service.rb           |  39 ----
 app/services/subscribe_service.rb                  |  58 -----
 app/services/unblock_service.rb                    |  12 +-
 app/services/unfavourite_service.rb                |  13 +-
 app/services/unfollow_service.rb                   |  16 +-
 app/services/unsubscribe_service.rb                |  36 ---
 app/services/update_remote_profile_service.rb      |  66 ------
 app/services/verify_salmon_service.rb              |  26 ---
 app/views/accounts/show.html.haml                  |   1 -
 .../admin/subscriptions/_subscription.html.haml    |  18 --
 app/views/admin/subscriptions/index.html.haml      |  16 --
 app/views/well_known/webfinger/show.xml.ruby       |   5 -
 app/workers/after_remote_follow_request_worker.rb  |  24 +-
 app/workers/after_remote_follow_worker.rb          |  24 +-
 app/workers/notification_worker.rb                 |   4 +-
 app/workers/processing_worker.rb                   |   4 +-
 app/workers/pubsubhubbub/confirmation_worker.rb    |  75 +-----
 app/workers/pubsubhubbub/delivery_worker.rb        |  74 +-----
 app/workers/pubsubhubbub/distribution_worker.rb    |  25 +-
 .../pubsubhubbub/raw_distribution_worker.rb        |  15 +-
 app/workers/pubsubhubbub/subscribe_worker.rb       |  27 +--
 app/workers/pubsubhubbub/unsubscribe_worker.rb     |   8 +-
 app/workers/remote_profile_update_worker.rb        |   6 +-
 app/workers/salmon_worker.rb                       |   6 +-
 app/workers/scheduler/subscriptions_scheduler.rb   |  10 +-
 config/locales/en.yml                              |   7 -
 config/navigation.rb                               |   1 -
 config/routes.rb                                   |  14 --
 config/sidekiq.yml                                 |   3 -
 spec/controllers/admin/accounts_controller_spec.rb |  38 ----
 .../admin/subscriptions_controller_spec.rb         |  32 ---
 spec/controllers/api/push_controller_spec.rb       |  59 -----
 spec/controllers/api/salmon_controller_spec.rb     |  65 ------
 .../api/subscriptions_controller_spec.rb           |  68 ------
 spec/controllers/api/v1/follows_controller_spec.rb |  51 -----
 spec/fixtures/requests/webfinger.txt               |   2 +-
 spec/lib/ostatus/atom_serializer_spec.rb           | 145 ------------
 spec/services/authorize_follow_service_spec.rb     |   7 -
 .../services/batched_remove_status_service_spec.rb |  13 --
 spec/services/block_service_spec.rb                |   7 -
 spec/services/favourite_service_spec.rb            |   7 -
 spec/services/fetch_remote_account_service_spec.rb |  40 ----
 spec/services/follow_service_spec.rb               |  68 ------
 spec/services/import_service_spec.rb               |  30 ++-
 spec/services/post_status_service_spec.rb          |   2 -
 spec/services/process_feed_service_spec.rb         | 252 ---------------------
 spec/services/process_interaction_service_spec.rb  | 151 ------------
 spec/services/process_mentions_service_spec.rb     |   4 -
 .../pubsubhubbub/subscribe_service_spec.rb         |  71 ------
 .../pubsubhubbub/unsubscribe_service_spec.rb       |  46 ----
 spec/services/reblog_service_spec.rb               |   4 -
 spec/services/reject_follow_service_spec.rb        |   7 -
 spec/services/remove_status_service_spec.rb        |  13 --
 spec/services/resolve_account_service_spec.rb      |  88 +------
 spec/services/send_interaction_service_spec.rb     |   7 -
 spec/services/subscribe_service_spec.rb            |  43 ----
 spec/services/unblock_service_spec.rb              |   7 -
 spec/services/unfollow_service_spec.rb             |   7 -
 spec/services/unsubscribe_service_spec.rb          |  37 ---
 .../services/update_remote_profile_service_spec.rb |  84 -------
 .../after_remote_follow_request_worker_spec.rb     |  59 -----
 spec/workers/after_remote_follow_worker_spec.rb    |  59 -----
 .../pubsubhubbub/confirmation_worker_spec.rb       |  88 -------
 spec/workers/pubsubhubbub/delivery_worker_spec.rb  |  68 ------
 .../pubsubhubbub/distribution_worker_spec.rb       |  46 ----
 .../scheduler/subscriptions_scheduler_spec.rb      |  19 --
 102 files changed, 69 insertions(+), 3568 deletions(-)
 delete mode 100644 app/controllers/api/push_controller.rb
 delete mode 100644 app/controllers/api/salmon_controller.rb
 delete mode 100644 app/controllers/api/subscriptions_controller.rb
 delete mode 100644 app/controllers/api/v1/follows_controller.rb
 delete mode 100644 app/lib/ostatus/activity/base.rb
 delete mode 100644 app/lib/ostatus/activity/creation.rb
 delete mode 100644 app/lib/ostatus/activity/deletion.rb
 delete mode 100644 app/lib/ostatus/activity/general.rb
 delete mode 100644 app/lib/ostatus/activity/post.rb
 delete mode 100644 app/lib/ostatus/activity/remote.rb
 delete mode 100644 app/lib/ostatus/activity/share.rb
 delete mode 100644 app/services/concerns/author_extractor.rb
 delete mode 100644 app/services/concerns/stream_entry_renderer.rb
 delete mode 100644 app/services/process_feed_service.rb
 delete mode 100644 app/services/process_interaction_service.rb
 delete mode 100644 app/services/pubsubhubbub/subscribe_service.rb
 delete mode 100644 app/services/pubsubhubbub/unsubscribe_service.rb
 delete mode 100644 app/services/send_interaction_service.rb
 delete mode 100644 app/services/subscribe_service.rb
 delete mode 100644 app/services/unsubscribe_service.rb
 delete mode 100644 app/services/update_remote_profile_service.rb
 delete mode 100644 app/services/verify_salmon_service.rb
 delete mode 100644 app/views/admin/subscriptions/_subscription.html.haml
 delete mode 100644 app/views/admin/subscriptions/index.html.haml
 delete mode 100644 spec/controllers/admin/subscriptions_controller_spec.rb
 delete mode 100644 spec/controllers/api/push_controller_spec.rb
 delete mode 100644 spec/controllers/api/salmon_controller_spec.rb
 delete mode 100644 spec/controllers/api/subscriptions_controller_spec.rb
 delete mode 100644 spec/controllers/api/v1/follows_controller_spec.rb
 delete mode 100644 spec/services/process_feed_service_spec.rb
 delete mode 100644 spec/services/process_interaction_service_spec.rb
 delete mode 100644 spec/services/pubsubhubbub/subscribe_service_spec.rb
 delete mode 100644 spec/services/pubsubhubbub/unsubscribe_service_spec.rb
 delete mode 100644 spec/services/send_interaction_service_spec.rb
 delete mode 100644 spec/services/subscribe_service_spec.rb
 delete mode 100644 spec/services/unsubscribe_service_spec.rb
 delete mode 100644 spec/services/update_remote_profile_service_spec.rb
 delete mode 100644 spec/workers/after_remote_follow_request_worker_spec.rb
 delete mode 100644 spec/workers/after_remote_follow_worker_spec.rb
 delete mode 100644 spec/workers/pubsubhubbub/confirmation_worker_spec.rb
 delete mode 100644 spec/workers/pubsubhubbub/delivery_worker_spec.rb
 delete mode 100644 spec/workers/pubsubhubbub/distribution_worker_spec.rb
 delete mode 100644 spec/workers/scheduler/subscriptions_scheduler_spec.rb

(limited to 'spec')

diff --git a/app/controllers/activitypub/inboxes_controller.rb b/app/controllers/activitypub/inboxes_controller.rb
index a0b7532c2..e2cd8eaed 100644
--- a/app/controllers/activitypub/inboxes_controller.rb
+++ b/app/controllers/activitypub/inboxes_controller.rb
@@ -44,7 +44,6 @@ class ActivityPub::InboxesController < Api::BaseController
       ResolveAccountWorker.perform_async(signed_request_account.acct)
     end
 
-    Pubsubhubbub::UnsubscribeWorker.perform_async(signed_request_account.id) if signed_request_account.subscribed?
     DeliveryFailureTracker.track_inverse_success!(signed_request_account)
   end
 
diff --git a/app/controllers/admin/accounts_controller.rb b/app/controllers/admin/accounts_controller.rb
index 0c7760d77..2fa1dfe5f 100644
--- a/app/controllers/admin/accounts_controller.rb
+++ b/app/controllers/admin/accounts_controller.rb
@@ -2,8 +2,8 @@
 
 module Admin
   class AccountsController < BaseController
-    before_action :set_account, only: [:show, :subscribe, :unsubscribe, :redownload, :remove_avatar, :remove_header, :enable, :unsilence, :unsuspend, :memorialize, :approve, :reject]
-    before_action :require_remote_account!, only: [:subscribe, :unsubscribe, :redownload]
+    before_action :set_account, only: [:show, :redownload, :remove_avatar, :remove_header, :enable, :unsilence, :unsuspend, :memorialize, :approve, :reject]
+    before_action :require_remote_account!, only: [:redownload]
     before_action :require_local_account!, only: [:enable, :memorialize, :approve, :reject]
 
     def index
@@ -19,18 +19,6 @@ module Admin
       @warnings                = @account.targeted_account_warnings.latest.custom
     end
 
-    def subscribe
-      authorize @account, :subscribe?
-      Pubsubhubbub::SubscribeWorker.perform_async(@account.id)
-      redirect_to admin_account_path(@account.id)
-    end
-
-    def unsubscribe
-      authorize @account, :unsubscribe?
-      Pubsubhubbub::UnsubscribeWorker.perform_async(@account.id)
-      redirect_to admin_account_path(@account.id)
-    end
-
     def memorialize
       authorize @account, :memorialize?
       @account.memorialize!
diff --git a/app/controllers/api/push_controller.rb b/app/controllers/api/push_controller.rb
deleted file mode 100644
index e04d19125..000000000
--- a/app/controllers/api/push_controller.rb
+++ /dev/null
@@ -1,73 +0,0 @@
-# frozen_string_literal: true
-
-class Api::PushController < Api::BaseController
-  include SignatureVerification
-
-  def update
-    response, status = process_push_request
-    render plain: response, status: status
-  end
-
-  private
-
-  def process_push_request
-    case hub_mode
-    when 'subscribe'
-      Pubsubhubbub::SubscribeService.new.call(account_from_topic, hub_callback, hub_secret, hub_lease_seconds, verified_domain)
-    when 'unsubscribe'
-      Pubsubhubbub::UnsubscribeService.new.call(account_from_topic, hub_callback)
-    else
-      ["Unknown mode: #{hub_mode}", 422]
-    end
-  end
-
-  def hub_mode
-    params['hub.mode']
-  end
-
-  def hub_topic
-    params['hub.topic']
-  end
-
-  def hub_callback
-    params['hub.callback']
-  end
-
-  def hub_lease_seconds
-    params['hub.lease_seconds']
-  end
-
-  def hub_secret
-    params['hub.secret']
-  end
-
-  def account_from_topic
-    if hub_topic.present? && local_domain? && account_feed_path?
-      Account.find_local(hub_topic_params[:username])
-    end
-  end
-
-  def hub_topic_params
-    @_hub_topic_params ||= Rails.application.routes.recognize_path(hub_topic_uri.path)
-  end
-
-  def hub_topic_uri
-    @_hub_topic_uri ||= Addressable::URI.parse(hub_topic).normalize
-  end
-
-  def local_domain?
-    TagManager.instance.web_domain?(hub_topic_domain)
-  end
-
-  def verified_domain
-    return signed_request_account.domain if signed_request_account
-  end
-
-  def hub_topic_domain
-    hub_topic_uri.host + (hub_topic_uri.port ? ":#{hub_topic_uri.port}" : '')
-  end
-
-  def account_feed_path?
-    hub_topic_params[:controller] == 'accounts' && hub_topic_params[:action] == 'show' && hub_topic_params[:format] == 'atom'
-  end
-end
diff --git a/app/controllers/api/salmon_controller.rb b/app/controllers/api/salmon_controller.rb
deleted file mode 100644
index ac5f3268d..000000000
--- a/app/controllers/api/salmon_controller.rb
+++ /dev/null
@@ -1,37 +0,0 @@
-# frozen_string_literal: true
-
-class Api::SalmonController < Api::BaseController
-  include SignatureVerification
-
-  before_action :set_account
-  respond_to :txt
-
-  def update
-    if verify_payload?
-      process_salmon
-      head 202
-    elsif payload.present?
-      render plain: signature_verification_failure_reason, status: 401
-    else
-      head 400
-    end
-  end
-
-  private
-
-  def set_account
-    @account = Account.find(params[:id])
-  end
-
-  def payload
-    @_payload ||= request.body.read
-  end
-
-  def verify_payload?
-    payload.present? && VerifySalmonService.new.call(payload)
-  end
-
-  def process_salmon
-    SalmonWorker.perform_async(@account.id, payload.force_encoding('UTF-8'))
-  end
-end
diff --git a/app/controllers/api/subscriptions_controller.rb b/app/controllers/api/subscriptions_controller.rb
deleted file mode 100644
index 89007f3d6..000000000
--- a/app/controllers/api/subscriptions_controller.rb
+++ /dev/null
@@ -1,51 +0,0 @@
-# frozen_string_literal: true
-
-class Api::SubscriptionsController < Api::BaseController
-  before_action :set_account
-  respond_to :txt
-
-  def show
-    if subscription.valid?(params['hub.topic'])
-      @account.update(subscription_expires_at: future_expires)
-      render plain: encoded_challenge, status: 200
-    else
-      head 404
-    end
-  end
-
-  def update
-    if subscription.verify(body, request.headers['HTTP_X_HUB_SIGNATURE'])
-      ProcessingWorker.perform_async(@account.id, body.force_encoding('UTF-8'))
-    end
-
-    head 200
-  end
-
-  private
-
-  def subscription
-    @_subscription ||= @account.subscription(
-      api_subscription_url(@account.id)
-    )
-  end
-
-  def body
-    @_body ||= request.body.read
-  end
-
-  def encoded_challenge
-    HTMLEntities.new.encode(params['hub.challenge'])
-  end
-
-  def future_expires
-    Time.now.utc + lease_seconds_or_default
-  end
-
-  def lease_seconds_or_default
-    (params['hub.lease_seconds'] || 1.day).to_i.seconds
-  end
-
-  def set_account
-    @account = Account.find(params[:id])
-  end
-end
diff --git a/app/controllers/api/v1/follows_controller.rb b/app/controllers/api/v1/follows_controller.rb
deleted file mode 100644
index 5420c0533..000000000
--- a/app/controllers/api/v1/follows_controller.rb
+++ /dev/null
@@ -1,31 +0,0 @@
-# frozen_string_literal: true
-
-class Api::V1::FollowsController < Api::BaseController
-  before_action -> { doorkeeper_authorize! :follow, :'write:follows' }
-  before_action :require_user!
-
-  respond_to :json
-
-  def create
-    raise ActiveRecord::RecordNotFound if follow_params[:uri].blank?
-
-    @account = FollowService.new.call(current_user.account, target_uri).try(:target_account)
-
-    if @account.nil?
-      username, domain = target_uri.split('@')
-      @account         = Account.find_remote!(username, domain)
-    end
-
-    render json: @account, serializer: REST::AccountSerializer
-  end
-
-  private
-
-  def target_uri
-    follow_params[:uri].strip.gsub(/\A@/, '')
-  end
-
-  def follow_params
-    params.permit(:uri)
-  end
-end
diff --git a/app/lib/ostatus/activity/base.rb b/app/lib/ostatus/activity/base.rb
deleted file mode 100644
index db70f1998..000000000
--- a/app/lib/ostatus/activity/base.rb
+++ /dev/null
@@ -1,71 +0,0 @@
-# frozen_string_literal: true
-
-class OStatus::Activity::Base
-  include Redisable
-
-  def initialize(xml, account = nil, **options)
-    @xml     = xml
-    @account = account
-    @options = options
-  end
-
-  def status?
-    [:activity, :note, :comment].include?(type)
-  end
-
-  def verb
-    raw = @xml.at_xpath('./activity:verb', activity: OStatus::TagManager::AS_XMLNS).content
-    OStatus::TagManager::VERBS.key(raw)
-  rescue
-    :post
-  end
-
-  def type
-    raw = @xml.at_xpath('./activity:object-type', activity: OStatus::TagManager::AS_XMLNS).content
-    OStatus::TagManager::TYPES.key(raw)
-  rescue
-    :activity
-  end
-
-  def id
-    @xml.at_xpath('./xmlns:id', xmlns: OStatus::TagManager::XMLNS).content
-  end
-
-  def url
-    link = @xml.xpath('./xmlns:link[@rel="alternate"]', xmlns: OStatus::TagManager::XMLNS).find { |link_candidate| link_candidate['type'] == 'text/html' }
-    link.nil? ? nil : link['href']
-  end
-
-  def activitypub_uri
-    link = @xml.xpath('./xmlns:link[@rel="alternate"]', xmlns: OStatus::TagManager::XMLNS).find { |link_candidate| ['application/activity+json', 'application/ld+json; profile="https://www.w3.org/ns/activitystreams"'].include?(link_candidate['type']) }
-    link.nil? ? nil : link['href']
-  end
-
-  def activitypub_uri?
-    activitypub_uri.present?
-  end
-
-  private
-
-  def find_status(uri)
-    if OStatus::TagManager.instance.local_id?(uri)
-      local_id = OStatus::TagManager.instance.unique_tag_to_local_id(uri, 'Status')
-      return Status.find_by(id: local_id)
-    elsif ActivityPub::TagManager.instance.local_uri?(uri)
-      local_id = ActivityPub::TagManager.instance.uri_to_local_id(uri)
-      return Status.find_by(id: local_id)
-    end
-
-    Status.find_by(uri: uri)
-  end
-
-  def find_activitypub_status(uri, href)
-    tag_matches = /tag:([^,:]+)[^:]*:objectId=([\d]+)/.match(uri)
-    href_matches = %r{/users/([^/]+)}.match(href)
-
-    unless tag_matches.nil? || href_matches.nil?
-      uri = "https://#{tag_matches[1]}/users/#{href_matches[1]}/statuses/#{tag_matches[2]}"
-      Status.find_by(uri: uri)
-    end
-  end
-end
diff --git a/app/lib/ostatus/activity/creation.rb b/app/lib/ostatus/activity/creation.rb
deleted file mode 100644
index 60de712db..000000000
--- a/app/lib/ostatus/activity/creation.rb
+++ /dev/null
@@ -1,219 +0,0 @@
-# frozen_string_literal: true
-
-class OStatus::Activity::Creation < OStatus::Activity::Base
-  def perform
-    if redis.exists("delete_upon_arrival:#{@account.id}:#{id}")
-      Rails.logger.debug "Delete for status #{id} was queued, ignoring"
-      return [nil, false]
-    end
-
-    return [nil, false] if @account.suspended? || invalid_origin?
-
-    RedisLock.acquire(lock_options) do |lock|
-      if lock.acquired?
-        # Return early if status already exists in db
-        @status = find_status(id)
-        return [@status, false] unless @status.nil?
-        @status = process_status
-      else
-        raise Mastodon::RaceConditionError
-      end
-    end
-
-    [@status, true]
-  end
-
-  def process_status
-    Rails.logger.debug "Creating remote status #{id}"
-    cached_reblog = reblog
-    status = nil
-
-    # Skip if the reblogged status is not public
-    return if cached_reblog && !(cached_reblog.public_visibility? || cached_reblog.unlisted_visibility?)
-
-    media_attachments = save_media.take(4)
-
-    ApplicationRecord.transaction do
-      status = Status.create!(
-        uri: id,
-        url: url,
-        account: @account,
-        reblog: cached_reblog,
-        text: content,
-        spoiler_text: content_warning,
-        created_at: published,
-        override_timestamps: @options[:override_timestamps],
-        reply: thread?,
-        language: content_language,
-        visibility: visibility_scope,
-        conversation: find_or_create_conversation,
-        thread: thread? ? find_status(thread.first) || find_activitypub_status(thread.first, thread.second) : nil,
-        media_attachment_ids: media_attachments.map(&:id),
-        sensitive: sensitive?
-      )
-
-      save_mentions(status)
-      save_hashtags(status)
-      save_emojis(status)
-    end
-
-    if thread? && status.thread.nil? && Request.valid_url?(thread.second)
-      Rails.logger.debug "Trying to attach #{status.id} (#{id}) to #{thread.first}"
-      ThreadResolveWorker.perform_async(status.id, thread.second)
-    end
-
-    Rails.logger.debug "Queuing remote status #{status.id} (#{id}) for distribution"
-
-    LinkCrawlWorker.perform_async(status.id) unless status.spoiler_text?
-
-    # Only continue if the status is supposed to have arrived in real-time.
-    # Note that if @options[:override_timestamps] isn't set, the status
-    # may have a lower snowflake id than other existing statuses, potentially
-    # "hiding" it from paginated API calls
-    return status unless @options[:override_timestamps] || status.within_realtime_window?
-
-    DistributionWorker.perform_async(status.id)
-
-    status
-  end
-
-  def content
-    @xml.at_xpath('./xmlns:content', xmlns: OStatus::TagManager::XMLNS).content
-  end
-
-  def content_language
-    @xml.at_xpath('./xmlns:content', xmlns: OStatus::TagManager::XMLNS)['xml:lang']&.presence || 'en'
-  end
-
-  def content_warning
-    @xml.at_xpath('./xmlns:summary', xmlns: OStatus::TagManager::XMLNS)&.content || ''
-  end
-
-  def visibility_scope
-    @xml.at_xpath('./mastodon:scope', mastodon: OStatus::TagManager::MTDN_XMLNS)&.content&.to_sym || :public
-  end
-
-  def published
-    @xml.at_xpath('./xmlns:published', xmlns: OStatus::TagManager::XMLNS).content
-  end
-
-  def thread?
-    !@xml.at_xpath('./thr:in-reply-to', thr: OStatus::TagManager::THR_XMLNS).nil?
-  end
-
-  def thread
-    thr = @xml.at_xpath('./thr:in-reply-to', thr: OStatus::TagManager::THR_XMLNS)
-    [thr['ref'], thr['href']]
-  end
-
-  private
-
-  def sensitive?
-    # OStatus-specific convention (not standard)
-    @xml.xpath('./xmlns:category', xmlns: OStatus::TagManager::XMLNS).any? { |category| category['term'] == 'nsfw' }
-  end
-
-  def find_or_create_conversation
-    uri = @xml.at_xpath('./ostatus:conversation', ostatus: OStatus::TagManager::OS_XMLNS)&.attribute('ref')&.content
-    return if uri.nil?
-
-    if OStatus::TagManager.instance.local_id?(uri)
-      local_id = OStatus::TagManager.instance.unique_tag_to_local_id(uri, 'Conversation')
-      return Conversation.find_by(id: local_id)
-    end
-
-    Conversation.find_by(uri: uri) || Conversation.create!(uri: uri)
-  end
-
-  def save_mentions(parent)
-    processed_account_ids = []
-
-    @xml.xpath('./xmlns:link[@rel="mentioned"]', xmlns: OStatus::TagManager::XMLNS).each do |link|
-      next if [OStatus::TagManager::TYPES[:group], OStatus::TagManager::TYPES[:collection]].include? link['ostatus:object-type']
-
-      mentioned_account = account_from_href(link['href'])
-
-      next if mentioned_account.nil? || processed_account_ids.include?(mentioned_account.id)
-
-      mentioned_account.mentions.where(status: parent).first_or_create(status: parent)
-
-      # So we can skip duplicate mentions
-      processed_account_ids << mentioned_account.id
-    end
-  end
-
-  def save_hashtags(parent)
-    tags = @xml.xpath('./xmlns:category', xmlns: OStatus::TagManager::XMLNS).map { |category| category['term'] }.select(&:present?)
-    ProcessHashtagsService.new.call(parent, tags)
-  end
-
-  def save_media
-    do_not_download = DomainBlock.reject_media?(@account.domain)
-    media_attachments = []
-
-    @xml.xpath('./xmlns:link[@rel="enclosure"]', xmlns: OStatus::TagManager::XMLNS).each do |link|
-      next unless link['href']
-
-      media = MediaAttachment.where(status: nil, remote_url: link['href']).first_or_initialize(account: @account, status: nil, remote_url: link['href'])
-      parsed_url = Addressable::URI.parse(link['href']).normalize
-
-      next if !%w(http https).include?(parsed_url.scheme) || parsed_url.host.empty?
-
-      media.save
-      media_attachments << media
-
-      next if do_not_download
-
-      begin
-        media.file_remote_url = link['href']
-        media.save!
-      rescue ActiveRecord::RecordInvalid
-        next
-      end
-    end
-
-    media_attachments
-  end
-
-  def save_emojis(parent)
-    do_not_download = DomainBlock.reject_media?(parent.account.domain)
-
-    return if do_not_download
-
-    @xml.xpath('./xmlns:link[@rel="emoji"]', xmlns: OStatus::TagManager::XMLNS).each do |link|
-      next unless link['href'] && link['name']
-
-      shortcode = link['name'].delete(':')
-      emoji     = CustomEmoji.find_by(shortcode: shortcode, domain: parent.account.domain)
-
-      next unless emoji.nil?
-
-      emoji = CustomEmoji.new(shortcode: shortcode, domain: parent.account.domain)
-      emoji.image_remote_url = link['href']
-      emoji.save
-    end
-  end
-
-  def account_from_href(href)
-    url = Addressable::URI.parse(href).normalize
-
-    if TagManager.instance.web_domain?(url.host)
-      Account.find_local(url.path.gsub('/users/', ''))
-    else
-      Account.where(uri: href).or(Account.where(url: href)).first || FetchRemoteAccountService.new.call(href)
-    end
-  end
-
-  def invalid_origin?
-    return false unless id.start_with?('http') # Legacy IDs cannot be checked
-
-    needle = Addressable::URI.parse(id).normalized_host
-
-    !(needle.casecmp(@account.domain).zero? ||
-      needle.casecmp(Addressable::URI.parse(@account.remote_url.presence || @account.uri).normalized_host).zero?)
-  end
-
-  def lock_options
-    { redis: Redis.current, key: "create:#{id}" }
-  end
-end
diff --git a/app/lib/ostatus/activity/deletion.rb b/app/lib/ostatus/activity/deletion.rb
deleted file mode 100644
index c98f5ee0a..000000000
--- a/app/lib/ostatus/activity/deletion.rb
+++ /dev/null
@@ -1,16 +0,0 @@
-# frozen_string_literal: true
-
-class OStatus::Activity::Deletion < OStatus::Activity::Base
-  def perform
-    Rails.logger.debug "Deleting remote status #{id}"
-
-    status   = Status.find_by(uri: id, account: @account)
-    status ||= Status.find_by(uri: activitypub_uri, account: @account) if activitypub_uri?
-
-    if status.nil?
-      redis.setex("delete_upon_arrival:#{@account.id}:#{id}", 6 * 3_600, id)
-    else
-      RemoveStatusService.new.call(status)
-    end
-  end
-end
diff --git a/app/lib/ostatus/activity/general.rb b/app/lib/ostatus/activity/general.rb
deleted file mode 100644
index 8a6aabc33..000000000
--- a/app/lib/ostatus/activity/general.rb
+++ /dev/null
@@ -1,20 +0,0 @@
-# frozen_string_literal: true
-
-class OStatus::Activity::General < OStatus::Activity::Base
-  def specialize
-    special_class&.new(@xml, @account, @options)
-  end
-
-  private
-
-  def special_class
-    case verb
-    when :post
-      OStatus::Activity::Post
-    when :share
-      OStatus::Activity::Share
-    when :delete
-      OStatus::Activity::Deletion
-    end
-  end
-end
diff --git a/app/lib/ostatus/activity/post.rb b/app/lib/ostatus/activity/post.rb
deleted file mode 100644
index 755ed8656..000000000
--- a/app/lib/ostatus/activity/post.rb
+++ /dev/null
@@ -1,23 +0,0 @@
-# frozen_string_literal: true
-
-class OStatus::Activity::Post < OStatus::Activity::Creation
-  def perform
-    status, just_created = super
-
-    if just_created
-      status.mentions.includes(:account).each do |mention|
-        mentioned_account = mention.account
-        next unless mentioned_account.local?
-        NotifyService.new.call(mentioned_account, mention)
-      end
-    end
-
-    status
-  end
-
-  private
-
-  def reblog
-    nil
-  end
-end
diff --git a/app/lib/ostatus/activity/remote.rb b/app/lib/ostatus/activity/remote.rb
deleted file mode 100644
index 5b204b6d8..000000000
--- a/app/lib/ostatus/activity/remote.rb
+++ /dev/null
@@ -1,11 +0,0 @@
-# frozen_string_literal: true
-
-class OStatus::Activity::Remote < OStatus::Activity::Base
-  def perform
-    if activitypub_uri?
-      find_status(activitypub_uri) || FetchRemoteStatusService.new.call(url)
-    else
-      find_status(id) || FetchRemoteStatusService.new.call(url)
-    end
-  end
-end
diff --git a/app/lib/ostatus/activity/share.rb b/app/lib/ostatus/activity/share.rb
deleted file mode 100644
index 5ca601415..000000000
--- a/app/lib/ostatus/activity/share.rb
+++ /dev/null
@@ -1,26 +0,0 @@
-# frozen_string_literal: true
-
-class OStatus::Activity::Share < OStatus::Activity::Creation
-  def perform
-    return if reblog.nil?
-
-    status, just_created = super
-    NotifyService.new.call(reblog.account, status) if reblog.account.local? && just_created
-    status
-  end
-
-  def object
-    @xml.at_xpath('.//activity:object', activity: OStatus::TagManager::AS_XMLNS)
-  end
-
-  private
-
-  def reblog
-    return @reblog if defined? @reblog
-
-    original_status = OStatus::Activity::Remote.new(object).perform
-    return if original_status.nil?
-
-    @reblog = original_status.reblog? ? original_status.reblog : original_status
-  end
-end
diff --git a/app/lib/ostatus/atom_serializer.rb b/app/lib/ostatus/atom_serializer.rb
index 9a05d96cf..f5c0e85ca 100644
--- a/app/lib/ostatus/atom_serializer.rb
+++ b/app/lib/ostatus/atom_serializer.rb
@@ -53,8 +53,6 @@ class OStatus::AtomSerializer
     append_element(feed, 'link', nil, rel: :alternate, type: 'text/html', href: ::TagManager.instance.url_for(account))
     append_element(feed, 'link', nil, rel: :self, type: 'application/atom+xml', href: account_url(account, format: 'atom'))
     append_element(feed, 'link', nil, rel: :next, type: 'application/atom+xml', href: account_url(account, format: 'atom', max_id: stream_entries.last.id)) if stream_entries.size == 20
-    append_element(feed, 'link', nil, rel: :hub, href: api_push_url)
-    append_element(feed, 'link', nil, rel: :salmon, href: api_salmon_url(account.id))
 
     stream_entries.each do |stream_entry|
       feed << entry(stream_entry)
diff --git a/app/models/account.rb b/app/models/account.rb
index c588451fc..d6772eb98 100644
--- a/app/models/account.rb
+++ b/app/models/account.rb
@@ -164,8 +164,7 @@ class Account < ApplicationRecord
   end
 
   def refresh!
-    return if local?
-    ResolveAccountService.new.call(acct)
+    ResolveAccountService.new.call(acct) unless local?
   end
 
   def silenced?
diff --git a/app/serializers/webfinger_serializer.rb b/app/serializers/webfinger_serializer.rb
index 8c0b07702..4220f697e 100644
--- a/app/serializers/webfinger_serializer.rb
+++ b/app/serializers/webfinger_serializer.rb
@@ -18,7 +18,6 @@ class WebfingerSerializer < ActiveModel::Serializer
       { rel: 'http://webfinger.net/rel/profile-page', type: 'text/html', href: short_account_url(object) },
       { rel: 'http://schemas.google.com/g/2010#updates-from', type: 'application/atom+xml', href: account_url(object, format: 'atom') },
       { rel: 'self', type: 'application/activity+json', href: account_url(object) },
-      { rel: 'salmon', href: api_salmon_url(object.id) },
       { rel: 'magic-public-key', href: "data:application/magic-public-key,#{object.magic_key}" },
       { rel: 'http://ostatus.org/schema/1.0/subscribe', template: "#{authorize_interaction_url}?uri={uri}" },
     ]
diff --git a/app/services/authorize_follow_service.rb b/app/services/authorize_follow_service.rb
index 29b8700c7..49bef727e 100644
--- a/app/services/authorize_follow_service.rb
+++ b/app/services/authorize_follow_service.rb
@@ -11,25 +11,17 @@ class AuthorizeFollowService < BaseService
       follow_request.authorize!
     end
 
-    create_notification(follow_request) unless source_account.local?
+    create_notification(follow_request) if !source_account.local? && source_account.activitypub?
     follow_request
   end
 
   private
 
   def create_notification(follow_request)
-    if follow_request.account.ostatus?
-      NotificationWorker.perform_async(build_xml(follow_request), follow_request.target_account_id, follow_request.account_id)
-    elsif follow_request.account.activitypub?
-      ActivityPub::DeliveryWorker.perform_async(build_json(follow_request), follow_request.target_account_id, follow_request.account.inbox_url)
-    end
+    ActivityPub::DeliveryWorker.perform_async(build_json(follow_request), follow_request.target_account_id, follow_request.account.inbox_url)
   end
 
   def build_json(follow_request)
     Oj.dump(serialize_payload(follow_request, ActivityPub::AcceptFollowSerializer))
   end
-
-  def build_xml(follow_request)
-    OStatus::AtomSerializer.render(OStatus::AtomSerializer.new.authorize_follow_request_salmon(follow_request))
-  end
 end
diff --git a/app/services/batched_remove_status_service.rb b/app/services/batched_remove_status_service.rb
index e328b1739..cb66debc8 100644
--- a/app/services/batched_remove_status_service.rb
+++ b/app/services/batched_remove_status_service.rb
@@ -1,7 +1,6 @@
 # frozen_string_literal: true
 
 class BatchedRemoveStatusService < BaseService
-  include StreamEntryRenderer
   include Redisable
 
   # Delete given statuses and reblogs of them
@@ -18,10 +17,7 @@ class BatchedRemoveStatusService < BaseService
     @mentions = statuses.each_with_object({}) { |s, h| h[s.id] = s.active_mentions.includes(:account).to_a }
     @tags     = statuses.each_with_object({}) { |s, h| h[s.id] = s.tags.pluck(:name) }
 
-    @stream_entry_batches  = []
-    @salmon_batches        = []
-    @json_payloads         = statuses.each_with_object({}) { |s, h| h[s.id] = Oj.dump(event: :delete, payload: s.id.to_s) }
-    @activity_xml          = {}
+    @json_payloads = statuses.each_with_object({}) { |s, h| h[s.id] = Oj.dump(event: :delete, payload: s.id.to_s) }
 
     # Ensure that rendered XML reflects destroyed state
     statuses.each do |status|
@@ -39,28 +35,16 @@ class BatchedRemoveStatusService < BaseService
 
       unpush_from_home_timelines(account, account_statuses)
       unpush_from_list_timelines(account, account_statuses)
-
-      batch_stream_entries(account, account_statuses) if account.local?
     end
 
     # Cannot be batched
     statuses.each do |status|
       unpush_from_public_timelines(status)
-      batch_salmon_slaps(status) if status.local?
     end
-
-    Pubsubhubbub::RawDistributionWorker.push_bulk(@stream_entry_batches) { |batch| batch }
-    NotificationWorker.push_bulk(@salmon_batches) { |batch| batch }
   end
 
   private
 
-  def batch_stream_entries(account, statuses)
-    statuses.each do |status|
-      @stream_entry_batches << [build_xml(status.stream_entry), account.id]
-    end
-  end
-
   def unpush_from_home_timelines(account, statuses)
     recipients = account.followers_for_local_distribution.to_a
 
@@ -101,20 +85,4 @@ class BatchedRemoveStatusService < BaseService
       end
     end
   end
-
-  def batch_salmon_slaps(status)
-    return if @mentions[status.id].empty?
-
-    recipients = @mentions[status.id].map(&:account).reject(&:local?).select(&:ostatus?).uniq(&:domain).map(&:id)
-
-    recipients.each do |recipient_id|
-      @salmon_batches << [build_xml(status.stream_entry), status.account_id, recipient_id]
-    end
-  end
-
-  def build_xml(stream_entry)
-    return @activity_xml[stream_entry.id] if @activity_xml.key?(stream_entry.id)
-
-    @activity_xml[stream_entry.id] = stream_entry_to_xml(stream_entry)
-  end
 end
diff --git a/app/services/block_domain_service.rb b/app/services/block_domain_service.rb
index c6eef04d4..c5e5e5761 100644
--- a/app/services/block_domain_service.rb
+++ b/app/services/block_domain_service.rb
@@ -44,7 +44,6 @@ class BlockDomainService < BaseService
 
   def suspend_accounts!
     blocked_domain_accounts.without_suspended.reorder(nil).find_each do |account|
-      UnsubscribeService.new.call(account) if account.subscribed?
       SuspendAccountService.new.call(account, suspended_at: @domain_block.created_at)
     end
   end
diff --git a/app/services/block_service.rb b/app/services/block_service.rb
index 9050a4858..da06361c2 100644
--- a/app/services/block_service.rb
+++ b/app/services/block_service.rb
@@ -13,25 +13,17 @@ class BlockService < BaseService
     block = account.block!(target_account)
 
     BlockWorker.perform_async(account.id, target_account.id)
-    create_notification(block) unless target_account.local?
+    create_notification(block) if !target_account.local? && target_account.activitypub?
     block
   end
 
   private
 
   def create_notification(block)
-    if block.target_account.ostatus?
-      NotificationWorker.perform_async(build_xml(block), block.account_id, block.target_account_id)
-    elsif block.target_account.activitypub?
-      ActivityPub::DeliveryWorker.perform_async(build_json(block), block.account_id, block.target_account.inbox_url)
-    end
+    ActivityPub::DeliveryWorker.perform_async(build_json(block), block.account_id, block.target_account.inbox_url)
   end
 
   def build_json(block)
     Oj.dump(serialize_payload(block, ActivityPub::BlockSerializer))
   end
-
-  def build_xml(block)
-    OStatus::AtomSerializer.render(OStatus::AtomSerializer.new.block_salmon(block))
-  end
 end
diff --git a/app/services/concerns/author_extractor.rb b/app/services/concerns/author_extractor.rb
deleted file mode 100644
index c2419e9ec..000000000
--- a/app/services/concerns/author_extractor.rb
+++ /dev/null
@@ -1,23 +0,0 @@
-# frozen_string_literal: true
-
-module AuthorExtractor
-  def author_from_xml(xml, update_profile = true)
-    return nil if xml.nil?
-
-    # Try <email> for acct
-    acct = xml.at_xpath('./xmlns:author/xmlns:email', xmlns: OStatus::TagManager::XMLNS)&.content
-
-    # Try <name> + <uri>
-    if acct.blank?
-      username = xml.at_xpath('./xmlns:author/xmlns:name', xmlns: OStatus::TagManager::XMLNS)&.content
-      uri      = xml.at_xpath('./xmlns:author/xmlns:uri', xmlns: OStatus::TagManager::XMLNS)&.content
-
-      return nil if username.blank? || uri.blank?
-
-      domain = Addressable::URI.parse(uri).normalized_host
-      acct   = "#{username}@#{domain}"
-    end
-
-    ResolveAccountService.new.call(acct, update_profile: update_profile)
-  end
-end
diff --git a/app/services/concerns/stream_entry_renderer.rb b/app/services/concerns/stream_entry_renderer.rb
deleted file mode 100644
index 9f6c8a082..000000000
--- a/app/services/concerns/stream_entry_renderer.rb
+++ /dev/null
@@ -1,7 +0,0 @@
-# frozen_string_literal: true
-
-module StreamEntryRenderer
-  def stream_entry_to_xml(stream_entry)
-    OStatus::AtomSerializer.render(OStatus::AtomSerializer.new.entry(stream_entry, true))
-  end
-end
diff --git a/app/services/favourite_service.rb b/app/services/favourite_service.rb
index 128a24ad6..02b26458a 100644
--- a/app/services/favourite_service.rb
+++ b/app/services/favourite_service.rb
@@ -30,8 +30,6 @@ class FavouriteService < BaseService
 
     if status.account.local?
       NotifyService.new.call(status.account, favourite)
-    elsif status.account.ostatus?
-      NotificationWorker.perform_async(build_xml(favourite), favourite.account_id, status.account_id)
     elsif status.account.activitypub?
       ActivityPub::DeliveryWorker.perform_async(build_json(favourite), favourite.account_id, status.account.inbox_url)
     end
@@ -46,8 +44,4 @@ class FavouriteService < BaseService
   def build_json(favourite)
     Oj.dump(serialize_payload(favourite, ActivityPub::LikeSerializer))
   end
-
-  def build_xml(favourite)
-    OStatus::AtomSerializer.render(OStatus::AtomSerializer.new.favourite_salmon(favourite))
-  end
 end
diff --git a/app/services/fetch_remote_account_service.rb b/app/services/fetch_remote_account_service.rb
index cfc560022..a7f95603d 100644
--- a/app/services/fetch_remote_account_service.rb
+++ b/app/services/fetch_remote_account_service.rb
@@ -1,8 +1,6 @@
 # frozen_string_literal: true
 
 class FetchRemoteAccountService < BaseService
-  include AuthorExtractor
-
   def call(url, prefetched_body = nil, protocol = :ostatus)
     if prefetched_body.nil?
       resource_url, resource_options, protocol = FetchAtomService.new.call(url)
@@ -12,34 +10,8 @@ class FetchRemoteAccountService < BaseService
     end
 
     case protocol
-    when :ostatus
-      process_atom(resource_url, **resource_options)
     when :activitypub
       ActivityPub::FetchRemoteAccountService.new.call(resource_url, **resource_options)
     end
   end
-
-  private
-
-  def process_atom(url, prefetched_body:)
-    xml = Nokogiri::XML(prefetched_body)
-    xml.encoding = 'utf-8'
-
-    account = author_from_xml(xml.at_xpath('/xmlns:feed', xmlns: OStatus::TagManager::XMLNS), false)
-
-    UpdateRemoteProfileService.new.call(xml, account) if account.present? && trusted_domain?(url, account)
-
-    account
-  rescue TypeError
-    Rails.logger.debug "Unparseable URL given: #{url}"
-    nil
-  rescue Nokogiri::XML::XPath::SyntaxError
-    Rails.logger.debug 'Invalid XML or missing namespace'
-    nil
-  end
-
-  def trusted_domain?(url, account)
-    domain = Addressable::URI.parse(url).normalized_host
-    domain.casecmp(account.domain).zero? || domain.casecmp(Addressable::URI.parse(account.remote_url.presence || account.uri).normalized_host).zero?
-  end
 end
diff --git a/app/services/fetch_remote_status_service.rb b/app/services/fetch_remote_status_service.rb
index 9c3008035..aac39dfd5 100644
--- a/app/services/fetch_remote_status_service.rb
+++ b/app/services/fetch_remote_status_service.rb
@@ -1,8 +1,6 @@
 # frozen_string_literal: true
 
 class FetchRemoteStatusService < BaseService
-  include AuthorExtractor
-
   def call(url, prefetched_body = nil, protocol = :ostatus)
     if prefetched_body.nil?
       resource_url, resource_options, protocol = FetchAtomService.new.call(url)
@@ -12,34 +10,8 @@ class FetchRemoteStatusService < BaseService
     end
 
     case protocol
-    when :ostatus
-      process_atom(resource_url, **resource_options)
     when :activitypub
       ActivityPub::FetchRemoteStatusService.new.call(resource_url, **resource_options)
     end
   end
-
-  private
-
-  def process_atom(url, prefetched_body:)
-    Rails.logger.debug "Processing Atom for remote status at #{url}"
-
-    xml = Nokogiri::XML(prefetched_body)
-    xml.encoding = 'utf-8'
-
-    account = author_from_xml(xml.at_xpath('/xmlns:entry', xmlns: OStatus::TagManager::XMLNS))
-    domain  = Addressable::URI.parse(url).normalized_host
-
-    return nil unless !account.nil? && confirmed_domain?(domain, account)
-
-    statuses = ProcessFeedService.new.call(prefetched_body, account)
-    statuses.first
-  rescue Nokogiri::XML::XPath::SyntaxError
-    Rails.logger.debug 'Invalid XML or missing namespace'
-    nil
-  end
-
-  def confirmed_domain?(domain, account)
-    account.domain.nil? || domain.casecmp(account.domain).zero? || domain.casecmp(Addressable::URI.parse(account.remote_url.presence || account.uri).normalized_host).zero?
-  end
 end
diff --git a/app/services/follow_service.rb b/app/services/follow_service.rb
index 0305e2d62..8e118f5d3 100644
--- a/app/services/follow_service.rb
+++ b/app/services/follow_service.rb
@@ -13,7 +13,7 @@ class FollowService < BaseService
     target_account = ResolveAccountService.new.call(target_account, skip_webfinger: true)
 
     raise ActiveRecord::RecordNotFound if target_account.nil? || target_account.id == source_account.id || target_account.suspended?
-    raise Mastodon::NotPermittedError  if target_account.blocking?(source_account) || source_account.blocking?(target_account) || target_account.moved?
+    raise Mastodon::NotPermittedError  if target_account.blocking?(source_account) || source_account.blocking?(target_account) || target_account.moved? || (!target_account.local? && target_account.ostatus?)
 
     if source_account.following?(target_account)
       # We're already following this account, but we'll call follow! again to
@@ -32,7 +32,7 @@ class FollowService < BaseService
 
     if target_account.locked? || target_account.activitypub?
       request_follow(source_account, target_account, reblogs: reblogs)
-    else
+    elsif target_account.local?
       direct_follow(source_account, target_account, reblogs: reblogs)
     end
   end
@@ -44,9 +44,6 @@ class FollowService < BaseService
 
     if target_account.local?
       LocalNotificationWorker.perform_async(target_account.id, follow_request.id, follow_request.class.name)
-    elsif target_account.ostatus?
-      NotificationWorker.perform_async(build_follow_request_xml(follow_request), source_account.id, target_account.id)
-      AfterRemoteFollowRequestWorker.perform_async(follow_request.id)
     elsif target_account.activitypub?
       ActivityPub::DeliveryWorker.perform_async(build_json(follow_request), source_account.id, target_account.inbox_url)
     end
@@ -57,27 +54,12 @@ class FollowService < BaseService
   def direct_follow(source_account, target_account, reblogs: true)
     follow = source_account.follow!(target_account, reblogs: reblogs)
 
-    if target_account.local?
-      LocalNotificationWorker.perform_async(target_account.id, follow.id, follow.class.name)
-    else
-      Pubsubhubbub::SubscribeWorker.perform_async(target_account.id) unless target_account.subscribed?
-      NotificationWorker.perform_async(build_follow_xml(follow), source_account.id, target_account.id)
-      AfterRemoteFollowWorker.perform_async(follow.id)
-    end
-
+    LocalNotificationWorker.perform_async(target_account.id, follow.id, follow.class.name)
     MergeWorker.perform_async(target_account.id, source_account.id)
 
     follow
   end
 
-  def build_follow_request_xml(follow_request)
-    OStatus::AtomSerializer.render(OStatus::AtomSerializer.new.follow_request_salmon(follow_request))
-  end
-
-  def build_follow_xml(follow)
-    OStatus::AtomSerializer.render(OStatus::AtomSerializer.new.follow_salmon(follow))
-  end
-
   def build_json(follow_request)
     Oj.dump(serialize_payload(follow_request, ActivityPub::FollowSerializer))
   end
diff --git a/app/services/post_status_service.rb b/app/services/post_status_service.rb
index 7830aee11..34ec6d504 100644
--- a/app/services/post_status_service.rb
+++ b/app/services/post_status_service.rb
@@ -88,7 +88,6 @@ class PostStatusService < BaseService
   def postprocess_status!
     LinkCrawlWorker.perform_async(@status.id) unless @status.spoiler_text?
     DistributionWorker.perform_async(@status.id)
-    Pubsubhubbub::DistributionWorker.perform_async(@status.stream_entry.id)
     ActivityPub::DistributionWorker.perform_async(@status.id)
     PollExpirationNotifyWorker.perform_at(@status.poll.expires_at, @status.poll.id) if @status.poll
   end
diff --git a/app/services/process_feed_service.rb b/app/services/process_feed_service.rb
deleted file mode 100644
index 30a9dd85e..000000000
--- a/app/services/process_feed_service.rb
+++ /dev/null
@@ -1,31 +0,0 @@
-# frozen_string_literal: true
-
-class ProcessFeedService < BaseService
-  def call(body, account, **options)
-    @options = options
-
-    xml = Nokogiri::XML(body)
-    xml.encoding = 'utf-8'
-
-    update_author(body, account)
-    process_entries(xml, account)
-  end
-
-  private
-
-  def update_author(body, account)
-    RemoteProfileUpdateWorker.perform_async(account.id, body.force_encoding('UTF-8'), true)
-  end
-
-  def process_entries(xml, account)
-    xml.xpath('//xmlns:entry', xmlns: OStatus::TagManager::XMLNS).reverse_each.map { |entry| process_entry(entry, account) }.compact
-  end
-
-  def process_entry(xml, account)
-    activity = OStatus::Activity::General.new(xml, account, @options)
-    activity.specialize&.perform if activity.status?
-  rescue ActiveRecord::RecordInvalid => e
-    Rails.logger.debug "Nothing was saved for #{activity.id} because: #{e}"
-    nil
-  end
-end
diff --git a/app/services/process_interaction_service.rb b/app/services/process_interaction_service.rb
deleted file mode 100644
index 1fca3832b..000000000
--- a/app/services/process_interaction_service.rb
+++ /dev/null
@@ -1,151 +0,0 @@
-# frozen_string_literal: true
-
-class ProcessInteractionService < BaseService
-  include AuthorExtractor
-  include Authorization
-
-  # Record locally the remote interaction with our user
-  # @param [String] envelope Salmon envelope
-  # @param [Account] target_account Account the Salmon was addressed to
-  def call(envelope, target_account)
-    body = salmon.unpack(envelope)
-
-    xml = Nokogiri::XML(body)
-    xml.encoding = 'utf-8'
-
-    account = author_from_xml(xml.at_xpath('/xmlns:entry', xmlns: OStatus::TagManager::XMLNS))
-
-    return if account.nil? || account.suspended?
-
-    if salmon.verify(envelope, account.keypair)
-      RemoteProfileUpdateWorker.perform_async(account.id, body.force_encoding('UTF-8'), true)
-
-      case verb(xml)
-      when :follow
-        follow!(account, target_account) unless target_account.locked? || target_account.blocking?(account) || target_account.domain_blocking?(account.domain)
-      when :request_friend
-        follow_request!(account, target_account) unless !target_account.locked? || target_account.blocking?(account) || target_account.domain_blocking?(account.domain)
-      when :authorize
-        authorize_follow_request!(account, target_account)
-      when :reject
-        reject_follow_request!(account, target_account)
-      when :unfollow
-        unfollow!(account, target_account)
-      when :favorite
-        favourite!(xml, account)
-      when :unfavorite
-        unfavourite!(xml, account)
-      when :post
-        add_post!(body, account) if mentions_account?(xml, target_account)
-      when :share
-        add_post!(body, account) unless status(xml).nil?
-      when :delete
-        delete_post!(xml, account)
-      when :block
-        reflect_block!(account, target_account)
-      when :unblock
-        reflect_unblock!(account, target_account)
-      end
-    end
-  rescue HTTP::Error, OStatus2::BadSalmonError, Mastodon::NotPermittedError
-    nil
-  end
-
-  private
-
-  def mentions_account?(xml, account)
-    xml.xpath('/xmlns:entry/xmlns:link[@rel="mentioned"]', xmlns: OStatus::TagManager::XMLNS).each { |mention_link| return true if [OStatus::TagManager.instance.uri_for(account), OStatus::TagManager.instance.url_for(account)].include?(mention_link.attribute('href').value) }
-    false
-  end
-
-  def verb(xml)
-    raw = xml.at_xpath('//activity:verb', activity: OStatus::TagManager::AS_XMLNS).content
-    OStatus::TagManager::VERBS.key(raw)
-  rescue
-    :post
-  end
-
-  def follow!(account, target_account)
-    follow = account.follow!(target_account)
-    FollowRequest.find_by(account: account, target_account: target_account)&.destroy
-    NotifyService.new.call(target_account, follow)
-  end
-
-  def follow_request!(account, target_account)
-    return if account.requested?(target_account)
-
-    follow_request = FollowRequest.create!(account: account, target_account: target_account)
-    NotifyService.new.call(target_account, follow_request)
-  end
-
-  def authorize_follow_request!(account, target_account)
-    follow_request = FollowRequest.find_by(account: target_account, target_account: account)
-    follow_request&.authorize!
-    Pubsubhubbub::SubscribeWorker.perform_async(account.id) unless account.subscribed?
-  end
-
-  def reject_follow_request!(account, target_account)
-    follow_request = FollowRequest.find_by(account: target_account, target_account: account)
-    follow_request&.reject!
-  end
-
-  def unfollow!(account, target_account)
-    account.unfollow!(target_account)
-    FollowRequest.find_by(account: account, target_account: target_account)&.destroy
-  end
-
-  def reflect_block!(account, target_account)
-    UnfollowService.new.call(target_account, account) if target_account.following?(account)
-    account.block!(target_account)
-  end
-
-  def reflect_unblock!(account, target_account)
-    UnblockService.new.call(account, target_account)
-  end
-
-  def delete_post!(xml, account)
-    status = Status.find(xml.at_xpath('//xmlns:id', xmlns: OStatus::TagManager::XMLNS).content)
-
-    return if status.nil?
-
-    authorize_with account, status, :destroy?
-
-    RemovalWorker.perform_async(status.id)
-  end
-
-  def favourite!(xml, from_account)
-    current_status = status(xml)
-
-    return if current_status.nil?
-
-    favourite = current_status.favourites.where(account: from_account).first_or_create!(account: from_account)
-    NotifyService.new.call(current_status.account, favourite)
-  end
-
-  def unfavourite!(xml, from_account)
-    current_status = status(xml)
-
-    return if current_status.nil?
-
-    favourite = current_status.favourites.where(account: from_account).first
-    favourite&.destroy
-  end
-
-  def add_post!(body, account)
-    ProcessingWorker.perform_async(account.id, body.force_encoding('UTF-8'))
-  end
-
-  def status(xml)
-    uri = activity_id(xml)
-    return nil unless OStatus::TagManager.instance.local_id?(uri)
-    Status.find(OStatus::TagManager.instance.unique_tag_to_local_id(uri, 'Status'))
-  end
-
-  def activity_id(xml)
-    xml.at_xpath('//activity:object', activity: OStatus::TagManager::AS_XMLNS).at_xpath('./xmlns:id', xmlns: OStatus::TagManager::XMLNS).content
-  end
-
-  def salmon
-    @salmon ||= OStatus2::Salmon.new
-  end
-end
diff --git a/app/services/process_mentions_service.rb b/app/services/process_mentions_service.rb
index bc607dff3..da52bff6a 100644
--- a/app/services/process_mentions_service.rb
+++ b/app/services/process_mentions_service.rb
@@ -1,7 +1,6 @@
 # frozen_string_literal: true
 
 class ProcessMentionsService < BaseService
-  include StreamEntryRenderer
   include Payloadable
 
   # Scan status for mentions and fetch remote mentioned users, create
@@ -49,17 +48,11 @@ class ProcessMentionsService < BaseService
 
     if mentioned_account.local?
       LocalNotificationWorker.perform_async(mentioned_account.id, mention.id, mention.class.name)
-    elsif mentioned_account.ostatus? && !@status.stream_entry.hidden?
-      NotificationWorker.perform_async(ostatus_xml, @status.account_id, mentioned_account.id)
     elsif mentioned_account.activitypub?
       ActivityPub::DeliveryWorker.perform_async(activitypub_json, mention.status.account_id, mentioned_account.inbox_url)
     end
   end
 
-  def ostatus_xml
-    @ostatus_xml ||= stream_entry_to_xml(@status.stream_entry)
-  end
-
   def activitypub_json
     return @activitypub_json if defined?(@activitypub_json)
     @activitypub_json = Oj.dump(serialize_payload(@status, ActivityPub::ActivitySerializer, signer: @status.account))
diff --git a/app/services/pubsubhubbub/subscribe_service.rb b/app/services/pubsubhubbub/subscribe_service.rb
deleted file mode 100644
index 550da6328..000000000
--- a/app/services/pubsubhubbub/subscribe_service.rb
+++ /dev/null
@@ -1,53 +0,0 @@
-# frozen_string_literal: true
-
-class Pubsubhubbub::SubscribeService < BaseService
-  URL_PATTERN = /\A#{URI.regexp(%w(http https))}\z/
-
-  attr_reader :account, :callback, :secret,
-              :lease_seconds, :domain
-
-  def call(account, callback, secret, lease_seconds, verified_domain = nil)
-    @account       = account
-    @callback      = Addressable::URI.parse(callback).normalize.to_s
-    @secret        = secret
-    @lease_seconds = lease_seconds
-    @domain        = verified_domain
-
-    process_subscribe
-  end
-
-  private
-
-  def process_subscribe
-    if account.nil?
-      ['Invalid topic URL', 422]
-    elsif !valid_callback?
-      ['Invalid callback URL', 422]
-    elsif blocked_domain?
-      ['Callback URL not allowed', 403]
-    else
-      confirm_subscription
-      ['', 202]
-    end
-  end
-
-  def confirm_subscription
-    subscription = locate_subscription
-    Pubsubhubbub::ConfirmationWorker.perform_async(subscription.id, 'subscribe', secret, lease_seconds)
-  end
-
-  def valid_callback?
-    callback.present? && callback =~ URL_PATTERN
-  end
-
-  def blocked_domain?
-    DomainBlock.blocked? Addressable::URI.parse(callback).host
-  end
-
-  def locate_subscription
-    subscription = Subscription.find_or_initialize_by(account: account, callback_url: callback)
-    subscription.domain = domain
-    subscription.save!
-    subscription
-  end
-end
diff --git a/app/services/pubsubhubbub/unsubscribe_service.rb b/app/services/pubsubhubbub/unsubscribe_service.rb
deleted file mode 100644
index 646150f7b..000000000
--- a/app/services/pubsubhubbub/unsubscribe_service.rb
+++ /dev/null
@@ -1,31 +0,0 @@
-# frozen_string_literal: true
-
-class Pubsubhubbub::UnsubscribeService < BaseService
-  attr_reader :account, :callback
-
-  def call(account, callback)
-    @account  = account
-    @callback = Addressable::URI.parse(callback).normalize.to_s
-
-    process_unsubscribe
-  end
-
-  private
-
-  def process_unsubscribe
-    if account.nil?
-      ['Invalid topic URL', 422]
-    else
-      confirm_unsubscribe unless subscription.nil?
-      ['', 202]
-    end
-  end
-
-  def confirm_unsubscribe
-    Pubsubhubbub::ConfirmationWorker.perform_async(subscription.id, 'unsubscribe')
-  end
-
-  def subscription
-    @_subscription ||= Subscription.find_by(account: account, callback_url: callback)
-  end
-end
diff --git a/app/services/reblog_service.rb b/app/services/reblog_service.rb
index 9cf4bc128..3bb460fca 100644
--- a/app/services/reblog_service.rb
+++ b/app/services/reblog_service.rb
@@ -2,7 +2,6 @@
 
 class ReblogService < BaseService
   include Authorization
-  include StreamEntryRenderer
   include Payloadable
 
   # Reblog a status and notify its remote author
@@ -24,7 +23,6 @@ class ReblogService < BaseService
     reblog = account.statuses.create!(reblog: reblogged_status, text: '', visibility: visibility)
 
     DistributionWorker.perform_async(reblog.id)
-    Pubsubhubbub::DistributionWorker.perform_async(reblog.stream_entry.id)
     ActivityPub::DistributionWorker.perform_async(reblog.id)
 
     create_notification(reblog)
@@ -40,8 +38,6 @@ class ReblogService < BaseService
 
     if reblogged_status.account.local?
       LocalNotificationWorker.perform_async(reblogged_status.account_id, reblog.id, reblog.class.name)
-    elsif reblogged_status.account.ostatus?
-      NotificationWorker.perform_async(stream_entry_to_xml(reblog.stream_entry), reblog.account_id, reblogged_status.account_id)
     elsif reblogged_status.account.activitypub? && !reblogged_status.account.following?(reblog.account)
       ActivityPub::DeliveryWorker.perform_async(build_json(reblog), reblog.account_id, reblogged_status.account.inbox_url)
     end
diff --git a/app/services/reject_follow_service.rb b/app/services/reject_follow_service.rb
index f87d0ba91..bc0000c8c 100644
--- a/app/services/reject_follow_service.rb
+++ b/app/services/reject_follow_service.rb
@@ -6,25 +6,17 @@ class RejectFollowService < BaseService
   def call(source_account, target_account)
     follow_request = FollowRequest.find_by!(account: source_account, target_account: target_account)
     follow_request.reject!
-    create_notification(follow_request) unless source_account.local?
+    create_notification(follow_request) if !source_account.local? && source_account.activitypub?
     follow_request
   end
 
   private
 
   def create_notification(follow_request)
-    if follow_request.account.ostatus?
-      NotificationWorker.perform_async(build_xml(follow_request), follow_request.target_account_id, follow_request.account_id)
-    elsif follow_request.account.activitypub?
-      ActivityPub::DeliveryWorker.perform_async(build_json(follow_request), follow_request.target_account_id, follow_request.account.inbox_url)
-    end
+    ActivityPub::DeliveryWorker.perform_async(build_json(follow_request), follow_request.target_account_id, follow_request.account.inbox_url)
   end
 
   def build_json(follow_request)
     Oj.dump(serialize_payload(follow_request, ActivityPub::RejectFollowSerializer))
   end
-
-  def build_xml(follow_request)
-    OStatus::AtomSerializer.render(OStatus::AtomSerializer.new.reject_follow_request_salmon(follow_request))
-  end
 end
diff --git a/app/services/remove_status_service.rb b/app/services/remove_status_service.rb
index 81adc5aae..a8c9100b3 100644
--- a/app/services/remove_status_service.rb
+++ b/app/services/remove_status_service.rb
@@ -1,7 +1,6 @@
 # frozen_string_literal: true
 
 class RemoveStatusService < BaseService
-  include StreamEntryRenderer
   include Redisable
   include Payloadable
 
@@ -78,11 +77,6 @@ class RemoveStatusService < BaseService
     target_accounts << @status.reblog.account if @status.reblog? && !@status.reblog.account.local?
     target_accounts.uniq!(&:id)
 
-    # Ostatus
-    NotificationWorker.push_bulk(target_accounts.select(&:ostatus?).uniq(&:domain)) do |target_account|
-      [salmon_xml, @account.id, target_account.id]
-    end
-
     # ActivityPub
     ActivityPub::DeliveryWorker.push_bulk(target_accounts.select(&:activitypub?).uniq(&:preferred_inbox_url)) do |target_account|
       [signed_activity_json, @account.id, target_account.preferred_inbox_url]
@@ -90,9 +84,6 @@ class RemoveStatusService < BaseService
   end
 
   def remove_from_remote_followers
-    # OStatus
-    Pubsubhubbub::RawDistributionWorker.perform_async(salmon_xml, @account.id)
-
     # ActivityPub
     ActivityPub::DeliveryWorker.push_bulk(@account.followers.inboxes) do |inbox_url|
       [signed_activity_json, @account.id, inbox_url]
@@ -111,10 +102,6 @@ class RemoveStatusService < BaseService
     end
   end
 
-  def salmon_xml
-    @salmon_xml ||= stream_entry_to_xml(@stream_entry)
-  end
-
   def signed_activity_json
     @signed_activity_json ||= Oj.dump(serialize_payload(@status, @status.reblog? ? ActivityPub::UndoAnnounceSerializer : ActivityPub::DeleteSerializer, signer: @account))
   end
diff --git a/app/services/resolve_account_service.rb b/app/services/resolve_account_service.rb
index e557706da..0ea31a0d8 100644
--- a/app/services/resolve_account_service.rb
+++ b/app/services/resolve_account_service.rb
@@ -1,11 +1,10 @@
 # frozen_string_literal: true
 
+require_relative '../models/account'
+
 class ResolveAccountService < BaseService
-  include OStatus2::MagicKey
   include JsonLdHelper
 
-  DFRN_NS = 'http://purl.org/macgirvin/dfrn/1.0'
-
   # Find or create a local account for a remote user.
   # When creating, look up the user's webfinger and fetch all
   # important information from their feed
@@ -48,18 +47,16 @@ class ResolveAccountService < BaseService
       return
     end
 
-    return if links_missing? || auto_suspend?
     return Account.find_local(@username) if TagManager.instance.local_domain?(@domain)
+    return unless activitypub_ready?
 
     RedisLock.acquire(lock_options) do |lock|
       if lock.acquired?
         @account = Account.find_remote(@username, @domain)
 
-        if activitypub_ready? || @account&.activitypub?
-          handle_activitypub
-        else
-          handle_ostatus
-        end
+        next unless @account.nil? || @account.activitypub?
+
+        handle_activitypub
       else
         raise Mastodon::RaceConditionError
       end
@@ -73,38 +70,12 @@ class ResolveAccountService < BaseService
 
   private
 
-  def links_missing?
-    !(activitypub_ready? || ostatus_ready?)
-  end
-
-  def ostatus_ready?
-    !(@webfinger.link('http://schemas.google.com/g/2010#updates-from').nil? ||
-      @webfinger.link('salmon').nil? ||
-      @webfinger.link('http://webfinger.net/rel/profile-page').nil? ||
-      @webfinger.link('magic-public-key').nil? ||
-      canonical_uri.nil? ||
-      hub_url.nil?)
-  end
-
   def webfinger_update_due?
     @account.nil? || ((!@options[:skip_webfinger] || @account.ostatus?) && @account.possibly_stale?)
   end
 
   def activitypub_ready?
-    !@webfinger.link('self').nil? &&
-      ['application/activity+json', 'application/ld+json; profile="https://www.w3.org/ns/activitystreams"'].include?(@webfinger.link('self').type) &&
-      !actor_json.nil? &&
-      actor_json['inbox'].present?
-  end
-
-  def handle_ostatus
-    create_account if @account.nil?
-    update_account
-    update_account_profile if update_profile?
-  end
-
-  def update_profile?
-    @options[:update_profile]
+    !@webfinger.link('self').nil? && ['application/activity+json', 'application/ld+json; profile="https://www.w3.org/ns/activitystreams"'].include?(@webfinger.link('self').type)
   end
 
   def handle_activitypub
@@ -115,89 +86,10 @@ class ResolveAccountService < BaseService
     nil
   end
 
-  def create_account
-    Rails.logger.debug "Creating new remote account for #{@username}@#{@domain}"
-
-    @account = Account.new(username: @username, domain: @domain)
-    @account.suspended_at = domain_block.created_at if auto_suspend?
-    @account.silenced_at  = domain_block.created_at if auto_silence?
-    @account.private_key  = nil
-  end
-
-  def update_account
-    @account.last_webfingered_at = Time.now.utc
-    @account.protocol            = :ostatus
-    @account.remote_url          = atom_url
-    @account.salmon_url          = salmon_url
-    @account.url                 = url
-    @account.public_key          = public_key
-    @account.uri                 = canonical_uri
-    @account.hub_url             = hub_url
-    @account.save!
-  end
-
-  def auto_suspend?
-    domain_block&.suspend?
-  end
-
-  def auto_silence?
-    domain_block&.silence?
-  end
-
-  def domain_block
-    return @domain_block if defined?(@domain_block)
-    @domain_block = DomainBlock.rule_for(@domain)
-  end
-
-  def atom_url
-    @atom_url ||= @webfinger.link('http://schemas.google.com/g/2010#updates-from').href
-  end
-
-  def salmon_url
-    @salmon_url ||= @webfinger.link('salmon').href
-  end
-
   def actor_url
     @actor_url ||= @webfinger.link('self').href
   end
 
-  def url
-    @url ||= @webfinger.link('http://webfinger.net/rel/profile-page').href
-  end
-
-  def public_key
-    @public_key ||= magic_key_to_pem(@webfinger.link('magic-public-key').href)
-  end
-
-  def canonical_uri
-    return @canonical_uri if defined?(@canonical_uri)
-
-    author_uri = atom.at_xpath('/xmlns:feed/xmlns:author/xmlns:uri')
-
-    if author_uri.nil?
-      owner      = atom.at_xpath('/xmlns:feed').at_xpath('./dfrn:owner', dfrn: DFRN_NS)
-      author_uri = owner.at_xpath('./xmlns:uri') unless owner.nil?
-    end
-
-    @canonical_uri = author_uri.nil? ? nil : author_uri.content
-  end
-
-  def hub_url
-    return @hub_url if defined?(@hub_url)
-
-    hubs     = atom.xpath('//xmlns:link[@rel="hub"]')
-    @hub_url = hubs.empty? || hubs.first['href'].nil? ? nil : hubs.first['href']
-  end
-
-  def atom_body
-    return @atom_body if defined?(@atom_body)
-
-    @atom_body = Request.new(:get, atom_url).perform do |response|
-      raise Mastodon::UnexpectedResponseError, response unless response.code == 200
-      response.body_with_limit
-    end
-  end
-
   def actor_json
     return @actor_json if defined?(@actor_json)
 
@@ -205,15 +97,6 @@ class ResolveAccountService < BaseService
     @actor_json = supported_context?(json) && equals_or_includes_any?(json['type'], ActivityPub::FetchRemoteAccountService::SUPPORTED_TYPES) ? json : nil
   end
 
-  def atom
-    return @atom if defined?(@atom)
-    @atom = Nokogiri::XML(atom_body)
-  end
-
-  def update_account_profile
-    RemoteProfileUpdateWorker.perform_async(@account.id, atom_body.force_encoding('UTF-8'), false)
-  end
-
   def lock_options
     { redis: Redis.current, key: "resolve:#{@username}@#{@domain}" }
   end
diff --git a/app/services/send_interaction_service.rb b/app/services/send_interaction_service.rb
deleted file mode 100644
index 3419043e5..000000000
--- a/app/services/send_interaction_service.rb
+++ /dev/null
@@ -1,39 +0,0 @@
-# frozen_string_literal: true
-
-class SendInteractionService < BaseService
-  # Send an Atom representation of an interaction to a remote Salmon endpoint
-  # @param [String] Entry XML
-  # @param [Account] source_account
-  # @param [Account] target_account
-  def call(xml, source_account, target_account)
-    @xml            = xml
-    @source_account = source_account
-    @target_account = target_account
-
-    return if !target_account.ostatus? || block_notification?
-
-    build_request.perform do |delivery|
-      raise Mastodon::UnexpectedResponseError, delivery unless delivery.code > 199 && delivery.code < 300
-    end
-  end
-
-  private
-
-  def build_request
-    request = Request.new(:post, @target_account.salmon_url, body: envelope)
-    request.add_headers('Content-Type' => 'application/magic-envelope+xml')
-    request
-  end
-
-  def envelope
-    salmon.pack(@xml, @source_account.keypair)
-  end
-
-  def block_notification?
-    DomainBlock.blocked?(@target_account.domain)
-  end
-
-  def salmon
-    @salmon ||= OStatus2::Salmon.new
-  end
-end
diff --git a/app/services/subscribe_service.rb b/app/services/subscribe_service.rb
deleted file mode 100644
index 83fd64396..000000000
--- a/app/services/subscribe_service.rb
+++ /dev/null
@@ -1,58 +0,0 @@
-# frozen_string_literal: true
-
-class SubscribeService < BaseService
-  def call(account)
-    return if account.hub_url.blank?
-
-    @account        = account
-    @account.secret = SecureRandom.hex
-
-    build_request.perform do |response|
-      if response_failed_permanently? response
-        # We're not allowed to subscribe. Fail and move on.
-        @account.secret = ''
-        @account.save!
-      elsif response_successful? response
-        # The subscription will be confirmed asynchronously.
-        @account.save!
-      else
-        # The response was either a 429 rate limit, or a 5xx error.
-        # We need to retry at a later time. Fail loudly!
-        raise Mastodon::UnexpectedResponseError, response
-      end
-    end
-  end
-
-  private
-
-  def build_request
-    request = Request.new(:post, @account.hub_url, form: subscription_params)
-    request.on_behalf_of(some_local_account) if some_local_account
-    request
-  end
-
-  def subscription_params
-    {
-      'hub.topic': @account.remote_url,
-      'hub.mode': 'subscribe',
-      'hub.callback': api_subscription_url(@account.id),
-      'hub.verify': 'async',
-      'hub.secret': @account.secret,
-      'hub.lease_seconds': 7.days.seconds,
-    }
-  end
-
-  def some_local_account
-    @some_local_account ||= Account.local.without_suspended.first
-  end
-
-  # Any response in the 3xx or 4xx range, except for 429 (rate limit)
-  def response_failed_permanently?(response)
-    (response.status.redirect? || response.status.client_error?) && !response.status.too_many_requests?
-  end
-
-  # Any response in the 2xx range
-  def response_successful?(response)
-    response.status.success?
-  end
-end
diff --git a/app/services/unblock_service.rb b/app/services/unblock_service.rb
index 95a858e9f..c263ac8af 100644
--- a/app/services/unblock_service.rb
+++ b/app/services/unblock_service.rb
@@ -7,25 +7,17 @@ class UnblockService < BaseService
     return unless account.blocking?(target_account)
 
     unblock = account.unblock!(target_account)
-    create_notification(unblock) unless target_account.local?
+    create_notification(unblock) if !target_account.local? && target_account.activitypub?
     unblock
   end
 
   private
 
   def create_notification(unblock)
-    if unblock.target_account.ostatus?
-      NotificationWorker.perform_async(build_xml(unblock), unblock.account_id, unblock.target_account_id)
-    elsif unblock.target_account.activitypub?
-      ActivityPub::DeliveryWorker.perform_async(build_json(unblock), unblock.account_id, unblock.target_account.inbox_url)
-    end
+    ActivityPub::DeliveryWorker.perform_async(build_json(unblock), unblock.account_id, unblock.target_account.inbox_url)
   end
 
   def build_json(unblock)
     Oj.dump(serialize_payload(unblock, ActivityPub::UndoBlockSerializer))
   end
-
-  def build_xml(block)
-    OStatus::AtomSerializer.render(OStatus::AtomSerializer.new.unblock_salmon(block))
-  end
 end
diff --git a/app/services/unfavourite_service.rb b/app/services/unfavourite_service.rb
index dcc890b7d..37917a64f 100644
--- a/app/services/unfavourite_service.rb
+++ b/app/services/unfavourite_service.rb
@@ -6,7 +6,7 @@ class UnfavouriteService < BaseService
   def call(account, status)
     favourite = Favourite.find_by!(account: account, status: status)
     favourite.destroy!
-    create_notification(favourite) unless status.local?
+    create_notification(favourite) if !status.account.local? && status.account.activitypub?
     favourite
   end
 
@@ -14,19 +14,10 @@ class UnfavouriteService < BaseService
 
   def create_notification(favourite)
     status = favourite.status
-
-    if status.account.ostatus?
-      NotificationWorker.perform_async(build_xml(favourite), favourite.account_id, status.account_id)
-    elsif status.account.activitypub?
-      ActivityPub::DeliveryWorker.perform_async(build_json(favourite), favourite.account_id, status.account.inbox_url)
-    end
+    ActivityPub::DeliveryWorker.perform_async(build_json(favourite), favourite.account_id, status.account.inbox_url)
   end
 
   def build_json(favourite)
     Oj.dump(serialize_payload(favourite, ActivityPub::UndoLikeSerializer))
   end
-
-  def build_xml(favourite)
-    OStatus::AtomSerializer.render(OStatus::AtomSerializer.new.unfavourite_salmon(favourite))
-  end
 end
diff --git a/app/services/unfollow_service.rb b/app/services/unfollow_service.rb
index 17dc29735..b7033d7eb 100644
--- a/app/services/unfollow_service.rb
+++ b/app/services/unfollow_service.rb
@@ -21,8 +21,8 @@ class UnfollowService < BaseService
     return unless follow
 
     follow.destroy!
-    create_notification(follow) unless @target_account.local?
-    create_reject_notification(follow) if @target_account.local? && !@source_account.local?
+    create_notification(follow) if !@target_account.local? && @target_account.activitypub?
+    create_reject_notification(follow) if @target_account.local? && !@source_account.local? && @source_account.activitypub?
     UnmergeWorker.perform_async(@target_account.id, @source_account.id)
     follow
   end
@@ -38,16 +38,10 @@ class UnfollowService < BaseService
   end
 
   def create_notification(follow)
-    if follow.target_account.ostatus?
-      NotificationWorker.perform_async(build_xml(follow), follow.account_id, follow.target_account_id)
-    elsif follow.target_account.activitypub?
-      ActivityPub::DeliveryWorker.perform_async(build_json(follow), follow.account_id, follow.target_account.inbox_url)
-    end
+    ActivityPub::DeliveryWorker.perform_async(build_json(follow), follow.account_id, follow.target_account.inbox_url)
   end
 
   def create_reject_notification(follow)
-    # Rejecting an already-existing follow request
-    return unless follow.account.activitypub?
     ActivityPub::DeliveryWorker.perform_async(build_reject_json(follow), follow.target_account_id, follow.account.inbox_url)
   end
 
@@ -58,8 +52,4 @@ class UnfollowService < BaseService
   def build_reject_json(follow)
     Oj.dump(serialize_payload(follow, ActivityPub::RejectFollowSerializer))
   end
-
-  def build_xml(follow)
-    OStatus::AtomSerializer.render(OStatus::AtomSerializer.new.unfollow_salmon(follow))
-  end
 end
diff --git a/app/services/unsubscribe_service.rb b/app/services/unsubscribe_service.rb
deleted file mode 100644
index 95c1fb4fc..000000000
--- a/app/services/unsubscribe_service.rb
+++ /dev/null
@@ -1,36 +0,0 @@
-# frozen_string_literal: true
-
-class UnsubscribeService < BaseService
-  def call(account)
-    return if account.hub_url.blank?
-
-    @account = account
-
-    begin
-      build_request.perform do |response|
-        Rails.logger.debug "PuSH unsubscribe for #{@account.acct} failed: #{response.status}" unless response.status.success?
-      end
-    rescue HTTP::Error, OpenSSL::SSL::SSLError => e
-      Rails.logger.debug "PuSH unsubscribe for #{@account.acct} failed: #{e}"
-    end
-
-    @account.secret = ''
-    @account.subscription_expires_at = nil
-    @account.save!
-  end
-
-  private
-
-  def build_request
-    Request.new(:post, @account.hub_url, form: subscription_params)
-  end
-
-  def subscription_params
-    {
-      'hub.topic': @account.remote_url,
-      'hub.mode': 'unsubscribe',
-      'hub.callback': api_subscription_url(@account.id),
-      'hub.verify': 'async',
-    }
-  end
-end
diff --git a/app/services/update_remote_profile_service.rb b/app/services/update_remote_profile_service.rb
deleted file mode 100644
index 403395a0d..000000000
--- a/app/services/update_remote_profile_service.rb
+++ /dev/null
@@ -1,66 +0,0 @@
-# frozen_string_literal: true
-
-class UpdateRemoteProfileService < BaseService
-  attr_reader :account, :remote_profile
-
-  def call(body, account, resubscribe = false)
-    @account        = account
-    @remote_profile = RemoteProfile.new(body)
-
-    return if remote_profile.root.nil?
-
-    update_account unless remote_profile.author.nil?
-
-    old_hub_url     = account.hub_url
-    account.hub_url = remote_profile.hub_link if remote_profile.hub_link.present? && remote_profile.hub_link != old_hub_url
-
-    account.save_with_optional_media!
-
-    Pubsubhubbub::SubscribeWorker.perform_async(account.id) if resubscribe && account.hub_url != old_hub_url
-  end
-
-  private
-
-  def update_account
-    account.display_name = remote_profile.display_name || ''
-    account.note         = remote_profile.note         || ''
-    account.locked       = remote_profile.locked?
-
-    if !account.suspended? && !DomainBlock.reject_media?(account.domain)
-      if remote_profile.avatar.present?
-        account.avatar_remote_url = remote_profile.avatar
-      else
-        account.avatar_remote_url = ''
-        account.avatar.destroy
-      end
-
-      if remote_profile.header.present?
-        account.header_remote_url = remote_profile.header
-      else
-        account.header_remote_url = ''
-        account.header.destroy
-      end
-
-      save_emojis if remote_profile.emojis.present?
-    end
-  end
-
-  def save_emojis
-    do_not_download = DomainBlock.reject_media?(account.domain)
-
-    return if do_not_download
-
-    remote_profile.emojis.each do |link|
-      next unless link['href'] && link['name']
-
-      shortcode = link['name'].delete(':')
-      emoji     = CustomEmoji.find_by(shortcode: shortcode, domain: account.domain)
-
-      next unless emoji.nil?
-
-      emoji = CustomEmoji.new(shortcode: shortcode, domain: account.domain)
-      emoji.image_remote_url = link['href']
-      emoji.save
-    end
-  end
-end
diff --git a/app/services/verify_salmon_service.rb b/app/services/verify_salmon_service.rb
deleted file mode 100644
index 205b35d8b..000000000
--- a/app/services/verify_salmon_service.rb
+++ /dev/null
@@ -1,26 +0,0 @@
-# frozen_string_literal: true
-
-class VerifySalmonService < BaseService
-  include AuthorExtractor
-
-  def call(payload)
-    body = salmon.unpack(payload)
-
-    xml = Nokogiri::XML(body)
-    xml.encoding = 'utf-8'
-
-    account = author_from_xml(xml.at_xpath('/xmlns:entry', xmlns: OStatus::TagManager::XMLNS))
-
-    if account.nil?
-      false
-    else
-      salmon.verify(payload, account.keypair)
-    end
-  end
-
-  private
-
-  def salmon
-    @salmon ||= OStatus2::Salmon.new
-  end
-end
diff --git a/app/views/accounts/show.html.haml b/app/views/accounts/show.html.haml
index 950e61847..de7d2a8ba 100644
--- a/app/views/accounts/show.html.haml
+++ b/app/views/accounts/show.html.haml
@@ -7,7 +7,6 @@
   - if @account.user&.setting_noindex
     %meta{ name: 'robots', content: 'noindex' }/
 
-  %link{ rel: 'salmon', href: api_salmon_url(@account.id) }/
   %link{ rel: 'alternate', type: 'application/atom+xml', href: account_url(@account, format: 'atom') }/
   %link{ rel: 'alternate', type: 'application/rss+xml', href: account_url(@account, format: 'rss') }/
   %link{ rel: 'alternate', type: 'application/activity+json', href: ActivityPub::TagManager.instance.uri_for(@account) }/
diff --git a/app/views/admin/subscriptions/_subscription.html.haml b/app/views/admin/subscriptions/_subscription.html.haml
deleted file mode 100644
index 1dec8e396..000000000
--- a/app/views/admin/subscriptions/_subscription.html.haml
+++ /dev/null
@@ -1,18 +0,0 @@
-%tr
-  %td
-    %samp= subscription.account.acct
-  %td
-    %samp= subscription.callback_url
-  %td
-    - if subscription.confirmed?
-      %i.fa.fa-check
-  %td{ style: "color: #{subscription.expired? ? 'red' : 'inherit'};" }
-    %time.time-ago{ datetime: subscription.expires_at.iso8601, title: l(subscription.expires_at) }
-      = precede subscription.expired? ? '-' : '' do
-        = time_ago_in_words(subscription.expires_at)
-  %td
-    - if subscription.last_successful_delivery_at?
-      %time.formatted{ datetime: subscription.last_successful_delivery_at.iso8601, title: l(subscription.last_successful_delivery_at) }
-        = l subscription.last_successful_delivery_at
-    - else
-      %i.fa.fa-times
diff --git a/app/views/admin/subscriptions/index.html.haml b/app/views/admin/subscriptions/index.html.haml
deleted file mode 100644
index 83704c8ee..000000000
--- a/app/views/admin/subscriptions/index.html.haml
+++ /dev/null
@@ -1,16 +0,0 @@
-- content_for :page_title do
-  = t('admin.subscriptions.title')
-
-.table-wrapper
-  %table.table
-    %thead
-      %tr
-        %th= t('admin.subscriptions.topic')
-        %th= t('admin.subscriptions.callback_url')
-        %th= t('admin.subscriptions.confirmed')
-        %th= t('admin.subscriptions.expires_in')
-        %th= t('admin.subscriptions.last_delivery')
-    %tbody
-      = render @subscriptions
-
-= paginate @subscriptions
diff --git a/app/views/well_known/webfinger/show.xml.ruby b/app/views/well_known/webfinger/show.xml.ruby
index 968c8c138..c82cdb7b3 100644
--- a/app/views/well_known/webfinger/show.xml.ruby
+++ b/app/views/well_known/webfinger/show.xml.ruby
@@ -25,11 +25,6 @@ doc << Ox::Element.new('XRD').tap do |xrd|
     link['href']     = account_url(@account)
   end
 
-  xrd << Ox::Element.new('Link').tap do |link|
-    link['rel']      = 'salmon'
-    link['href']     = api_salmon_url(@account.id)
-  end
-
   xrd << Ox::Element.new('Link').tap do |link|
     link['rel']      = 'magic-public-key'
     link['href']     = "data:application/magic-public-key,#{@account.magic_key}"
diff --git a/app/workers/after_remote_follow_request_worker.rb b/app/workers/after_remote_follow_request_worker.rb
index 84eb6ade2..ce9c65834 100644
--- a/app/workers/after_remote_follow_request_worker.rb
+++ b/app/workers/after_remote_follow_request_worker.rb
@@ -5,27 +5,5 @@ class AfterRemoteFollowRequestWorker
 
   sidekiq_options queue: 'pull', retry: 5
 
-  attr_reader :follow_request
-
-  def perform(follow_request_id)
-    @follow_request = FollowRequest.find(follow_request_id)
-    process_follow_service if processing_required?
-  rescue ActiveRecord::RecordNotFound
-    true
-  end
-
-  private
-
-  def process_follow_service
-    follow_request.destroy
-    FollowService.new.call(follow_request.account, updated_account.acct)
-  end
-
-  def processing_required?
-    !updated_account.nil? && !updated_account.locked?
-  end
-
-  def updated_account
-    @_updated_account ||= FetchRemoteAccountService.new.call(follow_request.target_account.remote_url)
-  end
+  def perform(follow_request_id); end
 end
diff --git a/app/workers/after_remote_follow_worker.rb b/app/workers/after_remote_follow_worker.rb
index edab83f85..d9719f2bf 100644
--- a/app/workers/after_remote_follow_worker.rb
+++ b/app/workers/after_remote_follow_worker.rb
@@ -5,27 +5,5 @@ class AfterRemoteFollowWorker
 
   sidekiq_options queue: 'pull', retry: 5
 
-  attr_reader :follow
-
-  def perform(follow_id)
-    @follow = Follow.find(follow_id)
-    process_follow_service if processing_required?
-  rescue ActiveRecord::RecordNotFound
-    true
-  end
-
-  private
-
-  def process_follow_service
-    follow.destroy
-    FollowService.new.call(follow.account, updated_account.acct)
-  end
-
-  def updated_account
-    @_updated_account ||= FetchRemoteAccountService.new.call(follow.target_account.remote_url)
-  end
-
-  def processing_required?
-    !updated_account.nil? && updated_account.locked?
-  end
+  def perform(follow_id); end
 end
diff --git a/app/workers/notification_worker.rb b/app/workers/notification_worker.rb
index da1d6ab45..1c0f001cf 100644
--- a/app/workers/notification_worker.rb
+++ b/app/workers/notification_worker.rb
@@ -5,7 +5,5 @@ class NotificationWorker
 
   sidekiq_options queue: 'push', retry: 5
 
-  def perform(xml, source_account_id, target_account_id)
-    SendInteractionService.new.call(xml, Account.find(source_account_id), Account.find(target_account_id))
-  end
+  def perform(xml, source_account_id, target_account_id); end
 end
diff --git a/app/workers/processing_worker.rb b/app/workers/processing_worker.rb
index 978c3aba2..cf3bd8397 100644
--- a/app/workers/processing_worker.rb
+++ b/app/workers/processing_worker.rb
@@ -5,7 +5,5 @@ class ProcessingWorker
 
   sidekiq_options backtrace: true
 
-  def perform(account_id, body)
-    ProcessFeedService.new.call(body, Account.find(account_id), override_timestamps: true)
-  end
+  def perform(account_id, body); end
 end
diff --git a/app/workers/pubsubhubbub/confirmation_worker.rb b/app/workers/pubsubhubbub/confirmation_worker.rb
index c0e7b677e..783a8c95f 100644
--- a/app/workers/pubsubhubbub/confirmation_worker.rb
+++ b/app/workers/pubsubhubbub/confirmation_worker.rb
@@ -2,81 +2,8 @@
 
 class Pubsubhubbub::ConfirmationWorker
   include Sidekiq::Worker
-  include RoutingHelper
 
   sidekiq_options queue: 'push', retry: false
 
-  attr_reader :subscription, :mode, :secret, :lease_seconds
-
-  def perform(subscription_id, mode, secret = nil, lease_seconds = nil)
-    @subscription = Subscription.find(subscription_id)
-    @mode = mode
-    @secret = secret
-    @lease_seconds = lease_seconds
-    process_confirmation
-  end
-
-  private
-
-  def process_confirmation
-    prepare_subscription
-
-    callback_get_with_params
-    logger.debug "Confirming PuSH subscription for #{subscription.callback_url} with challenge #{challenge}: #{@callback_response_body}"
-
-    update_subscription
-  end
-
-  def update_subscription
-    if successful_subscribe?
-      subscription.save!
-    elsif successful_unsubscribe?
-      subscription.destroy!
-    end
-  end
-
-  def successful_subscribe?
-    subscribing? && response_matches_challenge?
-  end
-
-  def successful_unsubscribe?
-    (unsubscribing? && response_matches_challenge?) || !subscription.confirmed?
-  end
-
-  def response_matches_challenge?
-    @callback_response_body == challenge
-  end
-
-  def subscribing?
-    mode == 'subscribe'
-  end
-
-  def unsubscribing?
-    mode == 'unsubscribe'
-  end
-
-  def callback_get_with_params
-    Request.new(:get, subscription.callback_url, params: callback_params).perform do |response|
-      @callback_response_body = response.body_with_limit
-    end
-  end
-
-  def callback_params
-    {
-      'hub.topic': account_url(subscription.account, format: :atom),
-      'hub.mode': mode,
-      'hub.challenge': challenge,
-      'hub.lease_seconds': subscription.lease_seconds,
-    }
-  end
-
-  def prepare_subscription
-    subscription.secret = secret
-    subscription.lease_seconds = lease_seconds
-    subscription.confirmed = true
-  end
-
-  def challenge
-    @_challenge ||= SecureRandom.hex
-  end
+  def perform(subscription_id, mode, secret = nil, lease_seconds = nil); end
 end
diff --git a/app/workers/pubsubhubbub/delivery_worker.rb b/app/workers/pubsubhubbub/delivery_worker.rb
index 619bfa48a..1260060bd 100644
--- a/app/workers/pubsubhubbub/delivery_worker.rb
+++ b/app/workers/pubsubhubbub/delivery_worker.rb
@@ -2,80 +2,8 @@
 
 class Pubsubhubbub::DeliveryWorker
   include Sidekiq::Worker
-  include RoutingHelper
 
   sidekiq_options queue: 'push', retry: 3, dead: false
 
-  sidekiq_retry_in do |count|
-    5 * (count + 1)
-  end
-
-  attr_reader :subscription, :payload
-
-  def perform(subscription_id, payload)
-    @subscription = Subscription.find(subscription_id)
-    @payload = payload
-    process_delivery unless blocked_domain?
-  rescue => e
-    raise e.class, "Delivery failed for #{subscription&.callback_url}: #{e.message}", e.backtrace[0]
-  end
-
-  private
-
-  def process_delivery
-    callback_post_payload do |payload_delivery|
-      raise Mastodon::UnexpectedResponseError, payload_delivery unless response_successful? payload_delivery
-    end
-
-    subscription.touch(:last_successful_delivery_at)
-  end
-
-  def callback_post_payload(&block)
-    request = Request.new(:post, subscription.callback_url, body: payload)
-    request.add_headers(headers)
-    request.perform(&block)
-  end
-
-  def blocked_domain?
-    DomainBlock.blocked?(host)
-  end
-
-  def host
-    Addressable::URI.parse(subscription.callback_url).normalized_host
-  end
-
-  def headers
-    {
-      'Content-Type' => 'application/atom+xml',
-      'Link' => link_header,
-    }.merge(signature_headers.to_h)
-  end
-
-  def link_header
-    LinkHeader.new([hub_link_header, self_link_header]).to_s
-  end
-
-  def hub_link_header
-    [api_push_url, [%w(rel hub)]]
-  end
-
-  def self_link_header
-    [account_url(subscription.account, format: :atom), [%w(rel self)]]
-  end
-
-  def signature_headers
-    { 'X-Hub-Signature' => payload_signature } if subscription.secret?
-  end
-
-  def payload_signature
-    "sha1=#{hmac_payload_digest}"
-  end
-
-  def hmac_payload_digest
-    OpenSSL::HMAC.hexdigest(OpenSSL::Digest.new('sha1'), subscription.secret, payload)
-  end
-
-  def response_successful?(payload_delivery)
-    payload_delivery.code > 199 && payload_delivery.code < 300
-  end
+  def perform(subscription_id, payload); end
 end
diff --git a/app/workers/pubsubhubbub/distribution_worker.rb b/app/workers/pubsubhubbub/distribution_worker.rb
index fed5e917d..75bac5d6f 100644
--- a/app/workers/pubsubhubbub/distribution_worker.rb
+++ b/app/workers/pubsubhubbub/distribution_worker.rb
@@ -5,28 +5,5 @@ class Pubsubhubbub::DistributionWorker
 
   sidekiq_options queue: 'push'
 
-  def perform(stream_entry_ids)
-    stream_entries = StreamEntry.where(id: stream_entry_ids).includes(:status).reject { |e| e.status.nil? || e.status.hidden? }
-
-    return if stream_entries.empty?
-
-    @account       = stream_entries.first.account
-    @subscriptions = active_subscriptions.to_a
-
-    distribute_public!(stream_entries)
-  end
-
-  private
-
-  def distribute_public!(stream_entries)
-    @payload = OStatus::AtomSerializer.render(OStatus::AtomSerializer.new.feed(@account, stream_entries))
-
-    Pubsubhubbub::DeliveryWorker.push_bulk(@subscriptions) do |subscription_id|
-      [subscription_id, @payload]
-    end
-  end
-
-  def active_subscriptions
-    Subscription.where(account: @account).active.pluck(:id)
-  end
+  def perform(stream_entry_ids); end
 end
diff --git a/app/workers/pubsubhubbub/raw_distribution_worker.rb b/app/workers/pubsubhubbub/raw_distribution_worker.rb
index 16962a623..ece9c80ac 100644
--- a/app/workers/pubsubhubbub/raw_distribution_worker.rb
+++ b/app/workers/pubsubhubbub/raw_distribution_worker.rb
@@ -5,18 +5,5 @@ class Pubsubhubbub::RawDistributionWorker
 
   sidekiq_options queue: 'push'
 
-  def perform(xml, source_account_id)
-    @account       = Account.find(source_account_id)
-    @subscriptions = active_subscriptions.to_a
-
-    Pubsubhubbub::DeliveryWorker.push_bulk(@subscriptions) do |subscription|
-      [subscription.id, xml]
-    end
-  end
-
-  private
-
-  def active_subscriptions
-    Subscription.where(account: @account).active.select('id, callback_url, domain')
-  end
+  def perform(xml, source_account_id); end
 end
diff --git a/app/workers/pubsubhubbub/subscribe_worker.rb b/app/workers/pubsubhubbub/subscribe_worker.rb
index 2e176d1c1..b861b5e67 100644
--- a/app/workers/pubsubhubbub/subscribe_worker.rb
+++ b/app/workers/pubsubhubbub/subscribe_worker.rb
@@ -5,30 +5,5 @@ class Pubsubhubbub::SubscribeWorker
 
   sidekiq_options queue: 'push', retry: 10, unique: :until_executed, dead: false
 
-  sidekiq_retry_in do |count|
-    case count
-    when 0
-      30.minutes.seconds
-    when 1
-      2.hours.seconds
-    when 2
-      12.hours.seconds
-    else
-      24.hours.seconds * (count - 2)
-    end
-  end
-
-  sidekiq_retries_exhausted do |msg, _e|
-    account = Account.find(msg['args'].first)
-    Sidekiq.logger.error "PuSH subscription attempts for #{account.acct} exhausted. Unsubscribing"
-    ::UnsubscribeService.new.call(account)
-  end
-
-  def perform(account_id)
-    account = Account.find(account_id)
-    logger.debug "PuSH re-subscribing to #{account.acct}"
-    ::SubscribeService.new.call(account)
-  rescue => e
-    raise e.class, "Subscribe failed for #{account&.acct}: #{e.message}", e.backtrace[0]
-  end
+  def perform(account_id); end
 end
diff --git a/app/workers/pubsubhubbub/unsubscribe_worker.rb b/app/workers/pubsubhubbub/unsubscribe_worker.rb
index a271715b7..0c1c263f6 100644
--- a/app/workers/pubsubhubbub/unsubscribe_worker.rb
+++ b/app/workers/pubsubhubbub/unsubscribe_worker.rb
@@ -5,11 +5,5 @@ class Pubsubhubbub::UnsubscribeWorker
 
   sidekiq_options queue: 'push', retry: false, unique: :until_executed, dead: false
 
-  def perform(account_id)
-    account = Account.find(account_id)
-    logger.debug "PuSH unsubscribing from #{account.acct}"
-    ::UnsubscribeService.new.call(account)
-  rescue ActiveRecord::RecordNotFound
-    true
-  end
+  def perform(account_id); end
 end
diff --git a/app/workers/remote_profile_update_worker.rb b/app/workers/remote_profile_update_worker.rb
index 03585ad2d..01e8daf8f 100644
--- a/app/workers/remote_profile_update_worker.rb
+++ b/app/workers/remote_profile_update_worker.rb
@@ -5,9 +5,5 @@ class RemoteProfileUpdateWorker
 
   sidekiq_options queue: 'pull'
 
-  def perform(account_id, body, resubscribe)
-    UpdateRemoteProfileService.new.call(body, Account.find(account_id), resubscribe)
-  rescue ActiveRecord::RecordNotFound
-    true
-  end
+  def perform(account_id, body, resubscribe); end
 end
diff --git a/app/workers/salmon_worker.rb b/app/workers/salmon_worker.rb
index d37d40432..10200b06c 100644
--- a/app/workers/salmon_worker.rb
+++ b/app/workers/salmon_worker.rb
@@ -5,9 +5,5 @@ class SalmonWorker
 
   sidekiq_options backtrace: true
 
-  def perform(account_id, body)
-    ProcessInteractionService.new.call(body, Account.find(account_id))
-  rescue Nokogiri::XML::XPath::SyntaxError, ActiveRecord::RecordNotFound
-    true
-  end
+  def perform(account_id, body); end
 end
diff --git a/app/workers/scheduler/subscriptions_scheduler.rb b/app/workers/scheduler/subscriptions_scheduler.rb
index d5873bccb..6903cadc7 100644
--- a/app/workers/scheduler/subscriptions_scheduler.rb
+++ b/app/workers/scheduler/subscriptions_scheduler.rb
@@ -5,13 +5,5 @@ class Scheduler::SubscriptionsScheduler
 
   sidekiq_options unique: :until_executed, retry: 0
 
-  def perform
-    Pubsubhubbub::SubscribeWorker.push_bulk(expiring_accounts.pluck(:id))
-  end
-
-  private
-
-  def expiring_accounts
-    Account.expiring(1.day.from_now).partitioned
-  end
+  def perform; end
 end
diff --git a/config/locales/en.yml b/config/locales/en.yml
index d4f1855aa..611f36fdd 100644
--- a/config/locales/en.yml
+++ b/config/locales/en.yml
@@ -469,13 +469,6 @@ en:
       no_status_selected: No statuses were changed as none were selected
       title: Account statuses
       with_media: With media
-    subscriptions:
-      callback_url: Callback URL
-      confirmed: Confirmed
-      expires_in: Expires in
-      last_delivery: Last delivery
-      title: WebSub
-      topic: Topic
     tags:
       accounts: Accounts
       hidden: Hidden
diff --git a/config/navigation.rb b/config/navigation.rb
index df1024189..ef845d1fc 100644
--- a/config/navigation.rb
+++ b/config/navigation.rb
@@ -48,7 +48,6 @@ SimpleNavigation::Configuration.run do |navigation|
       s.item :settings, safe_join([fa_icon('cogs fw'), t('admin.settings.title')]), edit_admin_settings_url, if: -> { current_user.admin? }, highlights_on: %r{/admin/settings}
       s.item :custom_emojis, safe_join([fa_icon('smile-o fw'), t('admin.custom_emojis.title')]), admin_custom_emojis_url, highlights_on: %r{/admin/custom_emojis}
       s.item :relays, safe_join([fa_icon('exchange fw'), t('admin.relays.title')]), admin_relays_url, if: -> { current_user.admin? }, highlights_on: %r{/admin/relays}
-      s.item :subscriptions, safe_join([fa_icon('paper-plane-o fw'), t('admin.subscriptions.title')]), admin_subscriptions_url, if: -> { current_user.admin? }
       s.item :sidekiq, safe_join([fa_icon('diamond fw'), 'Sidekiq']), sidekiq_url, link_html: { target: 'sidekiq' }, if: -> { current_user.admin? }
       s.item :pghero, safe_join([fa_icon('database fw'), 'PgHero']), pghero_url, link_html: { target: 'pghero' }, if: -> { current_user.admin? }
     end
diff --git a/config/routes.rb b/config/routes.rb
index 9ab5ba7f0..4b6d464c6 100644
--- a/config/routes.rb
+++ b/config/routes.rb
@@ -154,7 +154,6 @@ Rails.application.routes.draw do
   namespace :admin do
     get '/dashboard', to: 'dashboard#index'
 
-    resources :subscriptions, only: [:index]
     resources :domain_blocks, only: [:new, :create, :show, :destroy]
     resources :email_domain_blocks, only: [:index, :new, :create, :destroy]
     resources :action_logs, only: [:index]
@@ -191,8 +190,6 @@ Rails.application.routes.draw do
 
     resources :accounts, only: [:index, :show] do
       member do
-        post :subscribe
-        post :unsubscribe
         post :enable
         post :unsilence
         post :unsuspend
@@ -257,16 +254,6 @@ Rails.application.routes.draw do
   get '/admin', to: redirect('/admin/dashboard', status: 302)
 
   namespace :api do
-    # PubSubHubbub outgoing subscriptions
-    resources :subscriptions, only: [:show]
-    post '/subscriptions/:id', to: 'subscriptions#update'
-
-    # PubSubHubbub incoming subscriptions
-    post '/push', to: 'push#update', as: :push
-
-    # Salmon
-    post '/salmon/:id', to: 'salmon#update', as: :salmon
-
     # OEmbed
     get '/oembed', to: 'oembed#show', as: :oembed
 
@@ -318,7 +305,6 @@ Rails.application.routes.draw do
 
       get '/search', to: 'search#index', as: :search
 
-      resources :follows,      only: [:create]
       resources :media,        only: [:create, :update]
       resources :blocks,       only: [:index]
       resources :mutes,        only: [:index]
diff --git a/config/sidekiq.yml b/config/sidekiq.yml
index 0ec1742ab..a16dea967 100644
--- a/config/sidekiq.yml
+++ b/config/sidekiq.yml
@@ -9,9 +9,6 @@
   scheduled_statuses_scheduler:
     every: '5m'
     class: Scheduler::ScheduledStatusesScheduler
-  subscriptions_scheduler:
-    cron: '<%= Random.rand(0..59) %> <%= Random.rand(4..6) %> * * *'
-    class: Scheduler::SubscriptionsScheduler
   media_cleanup_scheduler:
     cron: '<%= Random.rand(0..59) %> <%= Random.rand(3..5) %> * * *'
     class: Scheduler::MediaCleanupScheduler
diff --git a/spec/controllers/admin/accounts_controller_spec.rb b/spec/controllers/admin/accounts_controller_spec.rb
index a348ab3d7..608606ff9 100644
--- a/spec/controllers/admin/accounts_controller_spec.rb
+++ b/spec/controllers/admin/accounts_controller_spec.rb
@@ -75,44 +75,6 @@ RSpec.describe Admin::AccountsController, type: :controller do
     end
   end
 
-  describe 'POST #subscribe' do
-    subject { post :subscribe, params: { id: account.id } }
-
-    let(:current_user) { Fabricate(:user, admin: admin) }
-    let(:account) { Fabricate(:account) }
-
-    context 'when user is admin' do
-      let(:admin) { true }
-
-      it { is_expected.to redirect_to admin_account_path(account.id) }
-    end
-
-    context 'when user is not admin' do
-      let(:admin) { false }
-
-      it { is_expected.to have_http_status :forbidden }
-    end
-  end
-
-  describe 'POST #unsubscribe' do
-    subject { post :unsubscribe, params: { id: account.id } }
-
-    let(:current_user) { Fabricate(:user, admin: admin) }
-    let(:account) { Fabricate(:account) }
-
-    context 'when user is admin' do
-      let(:admin) { true }
-
-      it { is_expected.to redirect_to admin_account_path(account.id) }
-    end
-
-    context 'when user is not admin' do
-      let(:admin) { false }
-
-      it { is_expected.to have_http_status :forbidden }
-    end
-  end
-
   describe 'POST #memorialize' do
     subject { post :memorialize, params: { id: account.id } }
 
diff --git a/spec/controllers/admin/subscriptions_controller_spec.rb b/spec/controllers/admin/subscriptions_controller_spec.rb
deleted file mode 100644
index 967152abe..000000000
--- a/spec/controllers/admin/subscriptions_controller_spec.rb
+++ /dev/null
@@ -1,32 +0,0 @@
-# frozen_string_literal: true
-require 'rails_helper'
-
-RSpec.describe Admin::SubscriptionsController, type: :controller do
-  render_views
-
-  describe 'GET #index' do
-    around do |example|
-      default_per_page = Subscription.default_per_page
-      Subscription.paginates_per 1
-      example.run
-      Subscription.paginates_per default_per_page
-    end
-
-    before do
-      sign_in Fabricate(:user, admin: true), scope: :user
-    end
-
-    it 'renders subscriptions' do
-      Fabricate(:subscription)
-      specified = Fabricate(:subscription)
-
-      get :index
-
-      subscriptions = assigns(:subscriptions)
-      expect(subscriptions.count).to eq 1
-      expect(subscriptions[0]).to eq specified
-
-      expect(response).to have_http_status(200)
-    end
-  end
-end
diff --git a/spec/controllers/api/push_controller_spec.rb b/spec/controllers/api/push_controller_spec.rb
deleted file mode 100644
index d769d8554..000000000
--- a/spec/controllers/api/push_controller_spec.rb
+++ /dev/null
@@ -1,59 +0,0 @@
-require 'rails_helper'
-
-RSpec.describe Api::PushController, type: :controller do
-  describe 'POST #update' do
-    context 'with hub.mode=subscribe' do
-      it 'creates a subscription' do
-        service = double(call: ['', 202])
-        allow(Pubsubhubbub::SubscribeService).to receive(:new).and_return(service)
-        account = Fabricate(:account)
-        account_topic_url = "https://#{Rails.configuration.x.local_domain}/users/#{account.username}.atom"
-        post :update, params: {
-          'hub.mode' => 'subscribe',
-          'hub.topic' => account_topic_url,
-          'hub.callback' => 'https://callback.host/api',
-          'hub.lease_seconds' => '3600',
-          'hub.secret' => 'as1234df',
-        }
-
-        expect(service).to have_received(:call).with(
-          account,
-          'https://callback.host/api',
-          'as1234df',
-          '3600',
-          nil
-        )
-        expect(response).to have_http_status(202)
-      end
-    end
-
-    context 'with hub.mode=unsubscribe' do
-      it 'unsubscribes the account' do
-        service = double(call: ['', 202])
-        allow(Pubsubhubbub::UnsubscribeService).to receive(:new).and_return(service)
-        account = Fabricate(:account)
-        account_topic_url = "https://#{Rails.configuration.x.local_domain}/users/#{account.username}.atom"
-        post :update, params: {
-          'hub.mode' => 'unsubscribe',
-          'hub.topic' => account_topic_url,
-          'hub.callback' => 'https://callback.host/api',
-        }
-
-        expect(service).to have_received(:call).with(
-          account,
-          'https://callback.host/api',
-        )
-        expect(response).to have_http_status(202)
-      end
-    end
-
-    context 'with unknown mode' do
-      it 'returns an unknown mode error' do
-        post :update, params: { 'hub.mode' => 'fake' }
-
-        expect(response).to have_http_status(422)
-        expect(response.body).to match(/Unknown mode/)
-      end
-    end
-  end
-end
diff --git a/spec/controllers/api/salmon_controller_spec.rb b/spec/controllers/api/salmon_controller_spec.rb
deleted file mode 100644
index 235a29af0..000000000
--- a/spec/controllers/api/salmon_controller_spec.rb
+++ /dev/null
@@ -1,65 +0,0 @@
-require 'rails_helper'
-
-RSpec.describe Api::SalmonController, type: :controller do
-  render_views
-
-  let(:account) { Fabricate(:user, account: Fabricate(:account, username: 'catsrgr8')).account }
-
-  before do
-    stub_request(:get, "https://quitter.no/.well-known/host-meta").to_return(request_fixture('.host-meta.txt'))
-    stub_request(:get, "https://quitter.no/.well-known/webfinger?resource=acct:gargron@quitter.no").to_return(request_fixture('webfinger.txt'))
-    stub_request(:get, "https://quitter.no/api/statuses/user_timeline/7477.atom").to_return(request_fixture('feed.txt'))
-    stub_request(:get, "https://quitter.no/avatar/7477-300-20160211190340.png").to_return(request_fixture('avatar.txt'))
-  end
-
-  describe 'POST #update' do
-    context 'with valid post data' do
-      before do
-        post :update, params: { id: account.id }, body: File.read(Rails.root.join('spec', 'fixtures', 'salmon', 'mention.xml'))
-      end
-
-      it 'contains XML in the request body' do
-        expect(request.body.read).to be_a String
-      end
-
-      it 'returns http success' do
-        expect(response).to have_http_status(202)
-      end
-
-      it 'creates remote account' do
-        expect(Account.find_by(username: 'gargron', domain: 'quitter.no')).to_not be_nil
-      end
-
-      it 'creates status' do
-        expect(Status.find_by(uri: 'tag:quitter.no,2016-03-20:noticeId=1276923:objectType=note')).to_not be_nil
-      end
-
-      it 'creates mention for target account' do
-        expect(account.mentions.count).to eq 1
-      end
-    end
-
-    context 'with empty post data' do
-      before do
-        post :update, params: { id: account.id }, body: ''
-      end
-
-      it 'returns http client error' do
-        expect(response).to have_http_status(400)
-      end
-    end
-
-    context 'with invalid post data' do
-      before do
-        service = double(call: false)
-        allow(VerifySalmonService).to receive(:new).and_return(service)
-
-        post :update, params: { id: account.id }, body: File.read(Rails.root.join('spec', 'fixtures', 'salmon', 'mention.xml'))
-      end
-
-      it 'returns http client error' do
-        expect(response).to have_http_status(401)
-      end
-    end
-  end
-end
diff --git a/spec/controllers/api/subscriptions_controller_spec.rb b/spec/controllers/api/subscriptions_controller_spec.rb
deleted file mode 100644
index 7a4252fe6..000000000
--- a/spec/controllers/api/subscriptions_controller_spec.rb
+++ /dev/null
@@ -1,68 +0,0 @@
-require 'rails_helper'
-
-RSpec.describe Api::SubscriptionsController, type: :controller do
-  render_views
-
-  let(:account) { Fabricate(:account, username: 'gargron', domain: 'quitter.no', remote_url: 'topic_url', secret: 'abc') }
-
-  describe 'GET #show' do
-    context 'with valid subscription' do
-      before do
-        get :show, params: { :id => account.id, 'hub.topic' => 'topic_url', 'hub.challenge' => '456', 'hub.lease_seconds' => "#{86400 * 30}" }
-      end
-
-      it 'returns http success' do
-        expect(response).to have_http_status(200)
-      end
-
-      it 'echoes back the challenge' do
-        expect(response.body).to match '456'
-      end
-    end
-
-    context 'with invalid subscription' do
-      before do
-        expect_any_instance_of(Account).to receive_message_chain(:subscription, :valid?).and_return(false)
-        get :show, params: { :id => account.id }
-      end
-
-      it 'returns http success' do
-        expect(response).to have_http_status(404)
-      end
-    end
-  end
-
-  describe 'POST #update' do
-    let(:feed) { File.read(Rails.root.join('spec', 'fixtures', 'push', 'feed.atom')) }
-
-    before do
-      stub_request(:post, "https://quitter.no/main/push/hub").to_return(:status => 200, :body => "", :headers => {})
-      stub_request(:get, "https://quitter.no/avatar/7477-300-20160211190340.png").to_return(request_fixture('avatar.txt'))
-      stub_request(:get, "https://quitter.no/notice/1269244").to_return(status: 404)
-      stub_request(:get, "https://quitter.no/notice/1265331").to_return(status: 404)
-      stub_request(:get, "https://community.highlandarrow.com/notice/54411").to_return(status: 404)
-      stub_request(:get, "https://community.highlandarrow.com/notice/53857").to_return(status: 404)
-      stub_request(:get, "https://community.highlandarrow.com/notice/51852").to_return(status: 404)
-      stub_request(:get, "https://social.umeahackerspace.se/notice/424348").to_return(status: 404)
-      stub_request(:get, "https://community.highlandarrow.com/notice/50467").to_return(status: 404)
-      stub_request(:get, "https://quitter.no/notice/1243309").to_return(status: 404)
-      stub_request(:get, "https://quitter.no/user/7477").to_return(status: 404)
-      stub_request(:any, "https://community.highlandarrow.com/user/1").to_return(status: 404)
-      stub_request(:any, "https://social.umeahackerspace.se/user/2").to_return(status: 404)
-      stub_request(:any, "https://gs.kawa-kun.com/user/2").to_return(status: 404)
-      stub_request(:any, "https://mastodon.social/users/Gargron").to_return(status: 404)
-
-      request.env['HTTP_X_HUB_SIGNATURE'] = "sha1=#{OpenSSL::HMAC.hexdigest('sha1', 'abc', feed)}"
-
-      post :update, params: { id: account.id }, body: feed
-    end
-
-    it 'returns http success' do
-      expect(response).to have_http_status(200)
-    end
-
-    it 'creates statuses for feed' do
-      expect(account.statuses.count).to_not eq 0
-    end
-  end
-end
diff --git a/spec/controllers/api/v1/follows_controller_spec.rb b/spec/controllers/api/v1/follows_controller_spec.rb
deleted file mode 100644
index 089e0fe5e..000000000
--- a/spec/controllers/api/v1/follows_controller_spec.rb
+++ /dev/null
@@ -1,51 +0,0 @@
-require 'rails_helper'
-
-RSpec.describe Api::V1::FollowsController, type: :controller do
-  render_views
-
-  let(:user)  { Fabricate(:user, account: Fabricate(:account, username: 'alice')) }
-  let(:token) { Fabricate(:accessible_access_token, resource_owner_id: user.id, scopes: 'write:follows') }
-
-  before do
-    allow(controller).to receive(:doorkeeper_token) { token }
-  end
-
-  describe 'POST #create' do
-    before do
-      stub_request(:get,  "https://quitter.no/.well-known/host-meta").to_return(request_fixture('.host-meta.txt'))
-      stub_request(:get,  "https://quitter.no/.well-known/webfinger?resource=acct:gargron@quitter.no").to_return(request_fixture('webfinger.txt'))
-      stub_request(:head, "https://quitter.no/api/statuses/user_timeline/7477.atom").to_return(:status => 405, :body => "", :headers => {})
-      stub_request(:get,  "https://quitter.no/api/statuses/user_timeline/7477.atom").to_return(request_fixture('feed.txt'))
-      stub_request(:get,  "https://quitter.no/avatar/7477-300-20160211190340.png").to_return(request_fixture('avatar.txt'))
-      stub_request(:post, "https://quitter.no/main/push/hub").to_return(:status => 200, :body => "", :headers => {})
-      stub_request(:post, "https://quitter.no/main/salmon/user/7477").to_return(:status => 200, :body => "", :headers => {})
-
-      post :create, params: { uri: 'gargron@quitter.no' }
-    end
-
-    it 'returns http success' do
-      expect(response).to have_http_status(200)
-    end
-
-    it 'creates account for remote user' do
-      expect(Account.find_by(username: 'gargron', domain: 'quitter.no')).to_not be_nil
-    end
-
-    it 'creates a follow relation between user and remote user' do
-      expect(user.account.following?(Account.find_by(username: 'gargron', domain: 'quitter.no'))).to be true
-    end
-
-    it 'sends a salmon slap to the remote user' do
-      expect(a_request(:post, "https://quitter.no/main/salmon/user/7477")).to have_been_made
-    end
-
-    it 'subscribes to remote hub' do
-      expect(a_request(:post, "https://quitter.no/main/push/hub")).to have_been_made
-    end
-
-    it 'returns http success if already following, too' do
-      post :create, params: { uri: 'gargron@quitter.no' }
-      expect(response).to have_http_status(200)
-    end
-  end
-end
diff --git a/spec/fixtures/requests/webfinger.txt b/spec/fixtures/requests/webfinger.txt
index edb8a2dbb..f337ecae6 100644
--- a/spec/fixtures/requests/webfinger.txt
+++ b/spec/fixtures/requests/webfinger.txt
@@ -8,4 +8,4 @@ Access-Control-Allow-Origin: *
 Vary: Accept-Encoding,Cookie
 Strict-Transport-Security: max-age=31536000; includeSubdomains;
 
-{"subject":"acct:gargron@quitter.no","aliases":["https:\/\/quitter.no\/user\/7477","https:\/\/quitter.no\/gargron","https:\/\/quitter.no\/index.php\/user\/7477","https:\/\/quitter.no\/index.php\/gargron"],"links":[{"rel":"http:\/\/webfinger.net\/rel\/profile-page","type":"text\/html","href":"https:\/\/quitter.no\/gargron"},{"rel":"http:\/\/gmpg.org\/xfn\/11","type":"text\/html","href":"https:\/\/quitter.no\/gargron"},{"rel":"describedby","type":"application\/rdf+xml","href":"https:\/\/quitter.no\/gargron\/foaf"},{"rel":"http:\/\/apinamespace.org\/atom","type":"application\/atomsvc+xml","href":"https:\/\/quitter.no\/api\/statusnet\/app\/service\/gargron.xml"},{"rel":"http:\/\/apinamespace.org\/twitter","href":"https:\/\/quitter.no\/api\/"},{"rel":"http:\/\/specs.openid.net\/auth\/2.0\/provider","href":"https:\/\/quitter.no\/gargron"},{"rel":"http:\/\/schemas.google.com\/g\/2010#updates-from","type":"application\/atom+xml","href":"https:\/\/quitter.no\/api\/statuses\/user_timeline\/7477.atom"},{"rel":"magic-public-key","href":"data:application\/magic-public-key,RSA.1ZBkHTavLvxH3FzlKv4O6WtlILKRFfNami3_Rcu8EuogtXSYiS-bB6hElZfUCSHbC4uLemOA34PEhz__CDMozax1iI_t8dzjDnh1x0iFSup7pSfW9iXk_WU3Dm74yWWW2jildY41vWgrEstuQ1dJ8vVFfSJ9T_tO4c-T9y8vDI8=.AQAB"},{"rel":"salmon","href":"https:\/\/quitter.no\/main\/salmon\/user\/7477"},{"rel":"http:\/\/salmon-protocol.org\/ns\/salmon-replies","href":"https:\/\/quitter.no\/main\/salmon\/user\/7477"},{"rel":"http:\/\/salmon-protocol.org\/ns\/salmon-mention","href":"https:\/\/quitter.no\/main\/salmon\/user\/7477"},{"rel":"http:\/\/ostatus.org\/schema\/1.0\/subscribe","template":"https:\/\/quitter.no\/main\/ostatussub?profile={uri}"}]}
\ No newline at end of file
+{"subject":"acct:gargron@quitter.no","aliases":["https:\/\/quitter.no\/user\/7477","https:\/\/quitter.no\/gargron","https:\/\/quitter.no\/index.php\/user\/7477","https:\/\/quitter.no\/index.php\/gargron"],"links":[{"rel":"http:\/\/webfinger.net\/rel\/profile-page","type":"text\/html","href":"https:\/\/quitter.no\/gargron"},{"rel":"http:\/\/gmpg.org\/xfn\/11","type":"text\/html","href":"https:\/\/quitter.no\/gargron"},{"rel":"describedby","type":"application\/rdf+xml","href":"https:\/\/quitter.no\/gargron\/foaf"},{"rel":"http:\/\/apinamespace.org\/atom","type":"application\/atomsvc+xml","href":"https:\/\/quitter.no\/api\/statusnet\/app\/service\/gargron.xml"},{"rel":"http:\/\/apinamespace.org\/twitter","href":"https:\/\/quitter.no\/api\/"},{"rel":"http:\/\/specs.openid.net\/auth\/2.0\/provider","href":"https:\/\/quitter.no\/gargron"},{"rel":"http:\/\/schemas.google.com\/g\/2010#updates-from","type":"application\/atom+xml","href":"https:\/\/quitter.no\/api\/statuses\/user_timeline\/7477.atom"},{"rel":"magic-public-key","href":"data:application\/magic-public-key,RSA.1ZBkHTavLvxH3FzlKv4O6WtlILKRFfNami3_Rcu8EuogtXSYiS-bB6hElZfUCSHbC4uLemOA34PEhz__CDMozax1iI_t8dzjDnh1x0iFSup7pSfW9iXk_WU3Dm74yWWW2jildY41vWgrEstuQ1dJ8vVFfSJ9T_tO4c-T9y8vDI8=.AQAB"},{"rel":"salmon","href":"https:\/\/quitter.no\/main\/salmon\/user\/7477"},{"rel":"http:\/\/salmon-protocol.org\/ns\/salmon-replies","href":"https:\/\/quitter.no\/main\/salmon\/user\/7477"},{"rel":"http:\/\/salmon-protocol.org\/ns\/salmon-mention","href":"https:\/\/quitter.no\/main\/salmon\/user\/7477"},{"rel":"http:\/\/ostatus.org\/schema\/1.0\/subscribe","template":"https:\/\/quitter.no\/main\/ostatussub?profile={uri}"}]}
diff --git a/spec/lib/ostatus/atom_serializer_spec.rb b/spec/lib/ostatus/atom_serializer_spec.rb
index 891871c1c..74ab7576f 100644
--- a/spec/lib/ostatus/atom_serializer_spec.rb
+++ b/spec/lib/ostatus/atom_serializer_spec.rb
@@ -406,28 +406,6 @@ RSpec.describe OStatus::AtomSerializer do
         scope = entry.nodes.find { |node| node.name == 'mastodon:scope' }
         expect(scope.text).to eq 'public'
       end
-
-      it 'returns element whose rendered view triggers creation when processed' do
-        remote_account = Account.create!(username: 'username')
-        remote_status = Fabricate(:status, account: remote_account, created_at: '2000-01-01T00:00:00Z')
-
-        entry = OStatus::AtomSerializer.new.entry(remote_status.stream_entry, true)
-        entry.nodes.delete_if { |node| node[:type] == 'application/activity+json' } # Remove ActivityPub link to simplify test
-        xml = OStatus::AtomSerializer.render(entry).gsub('cb6e6126.ngrok.io', 'remote.test')
-
-        remote_status.destroy!
-        remote_account.destroy!
-
-        account = Account.create!(
-          domain: 'remote.test',
-          username: 'username',
-          last_webfingered_at: Time.now.utc
-        )
-
-        ProcessFeedService.new.call(xml, account)
-
-        expect(Status.find_by(uri: "https://remote.test/users/#{remote_status.account.to_param}/statuses/#{remote_status.id}")).to be_instance_of Status
-      end
     end
 
     context 'if status is not present' do
@@ -683,24 +661,6 @@ RSpec.describe OStatus::AtomSerializer do
       end
     end
 
-    it 'appends link element for hub' do
-      account = Fabricate(:account, username: 'username')
-
-      feed = OStatus::AtomSerializer.new.feed(account, [])
-
-      link = feed.nodes.find { |node| node.name == 'link' && node[:rel] == 'hub' }
-      expect(link[:href]).to eq 'https://cb6e6126.ngrok.io/api/push'
-    end
-
-    it 'appends link element for Salmon' do
-      account = Fabricate(:account, username: 'username')
-
-      feed = OStatus::AtomSerializer.new.feed(account, [])
-
-      link = feed.nodes.find { |node| node.name == 'link' && node[:rel] == 'salmon' }
-      expect(link[:href]).to start_with 'https://cb6e6126.ngrok.io/api/salmon/'
-    end
-
     it 'appends stream entries' do
       account = Fabricate(:account, username: 'username')
       status = Fabricate(:status, account: account)
@@ -784,18 +744,6 @@ RSpec.describe OStatus::AtomSerializer do
       object = block_salmon.nodes.find { |node| node.name == 'activity:object' }
       expect(object.id.text).to eq 'https://domain.test/id'
     end
-
-    it 'returns element whose rendered view triggers block when processed' do
-      block = Fabricate(:block)
-      block_salmon = OStatus::AtomSerializer.new.block_salmon(block)
-      xml = OStatus::AtomSerializer.render(block_salmon)
-      envelope = OStatus2::Salmon.new.pack(xml, block.account.keypair)
-      block.destroy!
-
-      ProcessInteractionService.new.call(envelope, block.target_account)
-
-      expect(block.account.blocking?(block.target_account)).to be true
-    end
   end
 
   describe '#unblock_salmon' do
@@ -871,17 +819,6 @@ RSpec.describe OStatus::AtomSerializer do
       object = unblock_salmon.nodes.find { |node| node.name == 'activity:object' }
       expect(object.id.text).to eq 'https://domain.test/id'
     end
-
-    it 'returns element whose rendered view triggers block when processed' do
-      block = Fabricate(:block)
-      unblock_salmon = OStatus::AtomSerializer.new.unblock_salmon(block)
-      xml = OStatus::AtomSerializer.render(unblock_salmon)
-      envelope = OStatus2::Salmon.new.pack(xml, block.account.keypair)
-
-      ProcessInteractionService.new.call(envelope, block.target_account)
-
-      expect { block.reload }.to raise_error ActiveRecord::RecordNotFound
-    end
   end
 
   describe '#favourite_salmon' do
@@ -964,17 +901,6 @@ RSpec.describe OStatus::AtomSerializer do
       expect(favourite_salmon.title.text).to eq 'account favourited a status by status_account@remote'
       expect(favourite_salmon.content.text).to eq 'account favourited a status by status_account@remote'
     end
-
-    it 'returns element whose rendered view triggers favourite when processed' do
-      favourite = Fabricate(:favourite)
-      favourite_salmon = OStatus::AtomSerializer.new.favourite_salmon(favourite)
-      xml = OStatus::AtomSerializer.render(favourite_salmon)
-      envelope = OStatus2::Salmon.new.pack(xml, favourite.account.keypair)
-      favourite.destroy!
-
-      ProcessInteractionService.new.call(envelope, favourite.status.account)
-      expect(favourite.account.favourited?(favourite.status)).to be true
-    end
   end
 
   describe '#unfavourite_salmon' do
@@ -1064,16 +990,6 @@ RSpec.describe OStatus::AtomSerializer do
       expect(unfavourite_salmon.title.text).to eq 'account no longer favourites a status by status_account@remote'
       expect(unfavourite_salmon.content.text).to eq 'account no longer favourites a status by status_account@remote'
     end
-
-    it 'returns element whose rendered view triggers unfavourite when processed' do
-      favourite = Fabricate(:favourite)
-      unfavourite_salmon = OStatus::AtomSerializer.new.unfavourite_salmon(favourite)
-      xml = OStatus::AtomSerializer.render(unfavourite_salmon)
-      envelope = OStatus2::Salmon.new.pack(xml, favourite.account.keypair)
-
-      ProcessInteractionService.new.call(envelope, favourite.status.account)
-      expect { favourite.reload }.to raise_error ActiveRecord::RecordNotFound
-    end
   end
 
   describe '#follow_salmon' do
@@ -1143,18 +1059,6 @@ RSpec.describe OStatus::AtomSerializer do
       expect(follow_salmon.title.text).to eq 'account started following target_account@remote'
       expect(follow_salmon.content.text).to eq 'account started following target_account@remote'
     end
-
-    it 'returns element whose rendered view triggers follow when processed' do
-      follow = Fabricate(:follow)
-      follow_salmon = OStatus::AtomSerializer.new.follow_salmon(follow)
-      xml = OStatus::AtomSerializer.render(follow_salmon)
-      follow.destroy!
-      envelope = OStatus2::Salmon.new.pack(xml, follow.account.keypair)
-
-      ProcessInteractionService.new.call(envelope, follow.target_account)
-
-      expect(follow.account.following?(follow.target_account)).to be true
-    end
   end
 
   describe '#unfollow_salmon' do
@@ -1251,19 +1155,6 @@ RSpec.describe OStatus::AtomSerializer do
       object = unfollow_salmon.nodes.find { |node| node.name == 'activity:object' }
       expect(object.id.text).to eq 'https://domain.test/id'
     end
-
-    it 'returns element whose rendered view triggers unfollow when processed' do
-      follow = Fabricate(:follow)
-      follow.destroy!
-      unfollow_salmon = OStatus::AtomSerializer.new.unfollow_salmon(follow)
-      xml = OStatus::AtomSerializer.render(unfollow_salmon)
-      follow.account.follow!(follow.target_account)
-      envelope = OStatus2::Salmon.new.pack(xml, follow.account.keypair)
-
-      ProcessInteractionService.new.call(envelope, follow.target_account)
-
-      expect(follow.account.following?(follow.target_account)).to be false
-    end
   end
 
   describe '#follow_request_salmon' do
@@ -1294,18 +1185,6 @@ RSpec.describe OStatus::AtomSerializer do
         follow_request_salmon = serialize(follow_request)
         expect(follow_request_salmon.title.text).to eq 'account requested to follow target_account@remote'
       end
-
-      it 'returns element whose rendered view triggers follow request when processed' do
-        follow_request = Fabricate(:follow_request)
-        follow_request_salmon = serialize(follow_request)
-        xml = OStatus::AtomSerializer.render(follow_request_salmon)
-        envelope = OStatus2::Salmon.new.pack(xml, follow_request.account.keypair)
-        follow_request.destroy!
-
-        ProcessInteractionService.new.call(envelope, follow_request.target_account)
-
-        expect(follow_request.account.requested?(follow_request.target_account)).to eq true
-      end
     end
   end
 
@@ -1364,18 +1243,6 @@ RSpec.describe OStatus::AtomSerializer do
       verb = authorize_follow_request_salmon.nodes.find { |node| node.name == 'activity:verb' }
       expect(verb.text).to eq OStatus::TagManager::VERBS[:authorize]
     end
-
-    it 'returns element whose rendered view creates follow from follow request when processed' do
-      follow_request = Fabricate(:follow_request)
-      authorize_follow_request_salmon = OStatus::AtomSerializer.new.authorize_follow_request_salmon(follow_request)
-      xml = OStatus::AtomSerializer.render(authorize_follow_request_salmon)
-      envelope = OStatus2::Salmon.new.pack(xml, follow_request.target_account.keypair)
-
-      ProcessInteractionService.new.call(envelope, follow_request.account)
-
-      expect(follow_request.account.following?(follow_request.target_account)).to eq true
-      expect { follow_request.reload }.to raise_error ActiveRecord::RecordNotFound
-    end
   end
 
   describe '#reject_follow_request_salmon' do
@@ -1427,18 +1294,6 @@ RSpec.describe OStatus::AtomSerializer do
       verb = reject_follow_request_salmon.nodes.find { |node| node.name == 'activity:verb' }
       expect(verb.text).to eq OStatus::TagManager::VERBS[:reject]
     end
-
-    it 'returns element whose rendered view deletes follow request when processed' do
-      follow_request = Fabricate(:follow_request)
-      reject_follow_request_salmon = OStatus::AtomSerializer.new.reject_follow_request_salmon(follow_request)
-      xml = OStatus::AtomSerializer.render(reject_follow_request_salmon)
-      envelope = OStatus2::Salmon.new.pack(xml, follow_request.target_account.keypair)
-
-      ProcessInteractionService.new.call(envelope, follow_request.account)
-
-      expect(follow_request.account.following?(follow_request.target_account)).to eq false
-      expect { follow_request.reload }.to raise_error ActiveRecord::RecordNotFound
-    end
   end
 
   describe '#object' do
diff --git a/spec/services/authorize_follow_service_spec.rb b/spec/services/authorize_follow_service_spec.rb
index 562ef0041..ce56d57a6 100644
--- a/spec/services/authorize_follow_service_spec.rb
+++ b/spec/services/authorize_follow_service_spec.rb
@@ -38,13 +38,6 @@ RSpec.describe AuthorizeFollowService, type: :service do
     it 'creates follow relation' do
       expect(bob.following?(sender)).to be true
     end
-
-    it 'sends a follow request authorization salmon slap' do
-      expect(a_request(:post, "http://salmon.example.com/").with { |req|
-        xml = OStatus2::Salmon.new.unpack(req.body)
-        xml.match(OStatus::TagManager::VERBS[:authorize])
-      }).to have_been_made.once
-    end
   end
 
   describe 'remote ActivityPub' do
diff --git a/spec/services/batched_remove_status_service_spec.rb b/spec/services/batched_remove_status_service_spec.rb
index e53623449..d52e7f484 100644
--- a/spec/services/batched_remove_status_service_spec.rb
+++ b/spec/services/batched_remove_status_service_spec.rb
@@ -49,19 +49,6 @@ RSpec.describe BatchedRemoveStatusService, type: :service do
     expect(Redis.current).to have_received(:publish).with('timeline:public', any_args).at_least(:once)
   end
 
-  it 'sends PuSH update to PuSH subscribers' do
-    expect(a_request(:post, 'http://example.com/push').with { |req|
-      matches = req.body.match(OStatus::TagManager::VERBS[:delete])
-    }).to have_been_made.at_least_once
-  end
-
-  it 'sends Salmon slap to previously mentioned users' do
-    expect(a_request(:post, "http://example.com/salmon").with { |req|
-      xml = OStatus2::Salmon.new.unpack(req.body)
-      xml.match(OStatus::TagManager::VERBS[:delete])
-    }).to have_been_made.once
-  end
-
   it 'sends delete activity to followers' do
     expect(a_request(:post, 'http://example.com/inbox')).to have_been_made.at_least_once
   end
diff --git a/spec/services/block_service_spec.rb b/spec/services/block_service_spec.rb
index 6584bb90e..de20dd026 100644
--- a/spec/services/block_service_spec.rb
+++ b/spec/services/block_service_spec.rb
@@ -28,13 +28,6 @@ RSpec.describe BlockService, type: :service do
     it 'creates a blocking relation' do
       expect(sender.blocking?(bob)).to be true
     end
-
-    it 'sends a block salmon slap' do
-      expect(a_request(:post, "http://salmon.example.com/").with { |req|
-        xml = OStatus2::Salmon.new.unpack(req.body)
-        xml.match(OStatus::TagManager::VERBS[:block])
-      }).to have_been_made.once
-    end
   end
 
   describe 'remote ActivityPub' do
diff --git a/spec/services/favourite_service_spec.rb b/spec/services/favourite_service_spec.rb
index 0a20ccf6e..4c29ea77b 100644
--- a/spec/services/favourite_service_spec.rb
+++ b/spec/services/favourite_service_spec.rb
@@ -30,13 +30,6 @@ RSpec.describe FavouriteService, type: :service do
     it 'creates a favourite' do
       expect(status.favourites.first).to_not be_nil
     end
-
-    it 'sends a salmon slap' do
-      expect(a_request(:post, "http://salmon.example.com/").with { |req|
-        xml = OStatus2::Salmon.new.unpack(req.body)
-        xml.match(OStatus::TagManager::VERBS[:favorite])
-      }).to have_been_made.once
-    end
   end
 
   describe 'remote ActivityPub' do
diff --git a/spec/services/fetch_remote_account_service_spec.rb b/spec/services/fetch_remote_account_service_spec.rb
index 3cd86708b..37e9910d4 100644
--- a/spec/services/fetch_remote_account_service_spec.rb
+++ b/spec/services/fetch_remote_account_service_spec.rb
@@ -36,36 +36,6 @@ RSpec.describe FetchRemoteAccountService, type: :service do
     include_examples 'return Account'
   end
 
-  context 'protocol is :ostatus' do
-    let(:prefetched_body) { xml }
-    let(:protocol) { :ostatus }
-
-    before do
-      stub_request(:get, "https://kickass.zone/.well-known/webfinger?resource=acct:localhost@kickass.zone").to_return(request_fixture('webfinger-hacker3.txt'))
-      stub_request(:get, "https://kickass.zone/api/statuses/user_timeline/7477.atom").to_return(request_fixture('feed.txt'))
-    end
-
-    include_examples 'return Account'
-
-    it 'does not update account information if XML comes from an unverified domain' do
-      feed_xml = <<-XML.squish
-        <?xml version="1.0" encoding="UTF-8"?>
-        <feed xml:lang="en-US" xmlns="http://www.w3.org/2005/Atom" xmlns:thr="http://purl.org/syndication/thread/1.0" xmlns:georss="http://www.georss.org/georss" xmlns:activity="http://activitystrea.ms/spec/1.0/" xmlns:media="http://purl.org/syndication/atommedia" xmlns:poco="http://portablecontacts.net/spec/1.0" xmlns:ostatus="http://ostatus.org/schema/1.0" xmlns:statusnet="http://status.net/schema/api/1/">
-          <author>
-            <activity:object-type>http://activitystrea.ms/schema/1.0/person</activity:object-type>
-            <uri>http://kickass.zone/users/localhost</uri>
-            <name>localhost</name>
-            <poco:preferredUsername>localhost</poco:preferredUsername>
-            <poco:displayName>Villain!!!</poco:displayName>
-          </author>
-        </feed>
-      XML
-
-      returned_account = described_class.new.call('https://real-fake-domains.com/alice', feed_xml, :ostatus)
-      expect(returned_account.display_name).to_not eq 'Villain!!!'
-    end
-  end
-
   context 'when prefetched_body is nil' do
     context 'protocol is :activitypub' do
       before do
@@ -75,15 +45,5 @@ RSpec.describe FetchRemoteAccountService, type: :service do
 
       include_examples 'return Account'
     end
-
-    context 'protocol is :ostatus' do
-      before do
-        stub_request(:get, url).to_return(status: 200, body: xml, headers: { 'Content-Type' => 'application/atom+xml' })
-        stub_request(:get, "https://kickass.zone/.well-known/webfinger?resource=acct:localhost@kickass.zone").to_return(request_fixture('webfinger-hacker3.txt'))
-        stub_request(:get, "https://kickass.zone/api/statuses/user_timeline/7477.atom").to_return(request_fixture('feed.txt'))
-      end
-
-      include_examples 'return Account'
-    end
   end
 end
diff --git a/spec/services/follow_service_spec.rb b/spec/services/follow_service_spec.rb
index 3c4ec59be..86c85293e 100644
--- a/spec/services/follow_service_spec.rb
+++ b/spec/services/follow_service_spec.rb
@@ -96,74 +96,6 @@ RSpec.describe FollowService, type: :service do
     end
   end
 
-  context 'remote OStatus account' do
-    describe 'locked account' do
-      let(:bob) { Fabricate(:user, email: 'bob@example.com', account: Fabricate(:account, protocol: :ostatus, locked: true, username: 'bob', domain: 'example.com', salmon_url: 'http://salmon.example.com')).account }
-
-      before do
-        stub_request(:post, "http://salmon.example.com/").to_return(:status => 200, :body => "", :headers => {})
-        subject.call(sender, bob.acct)
-      end
-
-      it 'creates a follow request' do
-        expect(FollowRequest.find_by(account: sender, target_account: bob)).to_not be_nil
-      end
-
-      it 'sends a follow request salmon slap' do
-        expect(a_request(:post, "http://salmon.example.com/").with { |req|
-          xml = OStatus2::Salmon.new.unpack(req.body)
-          xml.match(OStatus::TagManager::VERBS[:request_friend])
-        }).to have_been_made.once
-      end
-    end
-
-    describe 'unlocked account' do
-      let(:bob) { Fabricate(:user, email: 'bob@example.com', account: Fabricate(:account, protocol: :ostatus, username: 'bob', domain: 'example.com', salmon_url: 'http://salmon.example.com', hub_url: 'http://hub.example.com')).account }
-
-      before do
-        stub_request(:post, "http://salmon.example.com/").to_return(:status => 200, :body => "", :headers => {})
-        stub_request(:post, "http://hub.example.com/").to_return(status: 202)
-        subject.call(sender, bob.acct)
-      end
-
-      it 'creates a following relation' do
-        expect(sender.following?(bob)).to be true
-      end
-
-      it 'sends a follow salmon slap' do
-        expect(a_request(:post, "http://salmon.example.com/").with { |req|
-          xml = OStatus2::Salmon.new.unpack(req.body)
-          xml.match(OStatus::TagManager::VERBS[:follow])
-        }).to have_been_made.once
-      end
-
-      it 'subscribes to PuSH' do
-        expect(a_request(:post, "http://hub.example.com/")).to have_been_made.once
-      end
-    end
-
-    describe 'already followed account' do
-      let(:bob) { Fabricate(:user, email: 'bob@example.com', account: Fabricate(:account, protocol: :ostatus, username: 'bob', domain: 'example.com', salmon_url: 'http://salmon.example.com', hub_url: 'http://hub.example.com')).account }
-
-      before do
-        sender.follow!(bob)
-        subject.call(sender, bob.acct)
-      end
-
-      it 'keeps a following relation' do
-        expect(sender.following?(bob)).to be true
-      end
-
-      it 'does not send a follow salmon slap' do
-        expect(a_request(:post, "http://salmon.example.com/")).not_to have_been_made
-      end
-
-      it 'does not subscribe to PuSH' do
-        expect(a_request(:post, "http://hub.example.com/")).not_to have_been_made
-      end
-    end
-  end
-
   context 'remote ActivityPub account' do
     let(:bob) { Fabricate(:user, account: Fabricate(:account, username: 'bob', domain: 'example.com', protocol: :activitypub, inbox_url: 'http://example.com/inbox')).account }
 
diff --git a/spec/services/import_service_spec.rb b/spec/services/import_service_spec.rb
index 5cf2dadf0..5355133f4 100644
--- a/spec/services/import_service_spec.rb
+++ b/spec/services/import_service_spec.rb
@@ -3,7 +3,11 @@ require 'rails_helper'
 RSpec.describe ImportService, type: :service do
   let!(:account) { Fabricate(:account, locked: false) }
   let!(:bob)     { Fabricate(:account, username: 'bob', locked: false) }
-  let!(:eve)     { Fabricate(:account, username: 'eve', domain: 'example.com', locked: false) }
+  let!(:eve)     { Fabricate(:account, username: 'eve', domain: 'example.com', locked: false, protocol: :activitypub, inbox_url: 'https://example.com/inbox') }
+
+  before do
+    stub_request(:post, "https://example.com/inbox").to_return(status: 200)
+  end
 
   context 'import old-style list of muted users' do
     subject { ImportService.new }
@@ -95,7 +99,8 @@ RSpec.describe ImportService, type: :service do
       let(:import) { Import.create(account: account, type: 'following', data: csv) }
       it 'follows the listed accounts, including boosts' do
         subject.call(import)
-        expect(account.following.count).to eq 2
+        expect(account.following.count).to eq 1
+        expect(account.follow_requests.count).to eq 1
         expect(Follow.find_by(account: account, target_account: bob).show_reblogs).to be true
       end
     end
@@ -106,7 +111,8 @@ RSpec.describe ImportService, type: :service do
       it 'follows the listed accounts, including notifications' do
         account.follow!(bob, reblogs: false)
         subject.call(import)
-        expect(account.following.count).to eq 2
+        expect(account.following.count).to eq 1
+        expect(account.follow_requests.count).to eq 1
         expect(Follow.find_by(account: account, target_account: bob).show_reblogs).to be true
       end
     end
@@ -117,7 +123,8 @@ RSpec.describe ImportService, type: :service do
       it 'mutes the listed accounts, including notifications' do
         account.follow!(bob, reblogs: false)
         subject.call(import)
-        expect(account.following.count).to eq 2
+        expect(account.following.count).to eq 1
+        expect(account.follow_requests.count).to eq 1
         expect(Follow.find_by(account: account, target_account: bob).show_reblogs).to be true
       end
     end
@@ -136,9 +143,10 @@ RSpec.describe ImportService, type: :service do
       let(:import) { Import.create(account: account, type: 'following', data: csv) }
       it 'follows the listed accounts, respecting boosts' do
         subject.call(import)
-        expect(account.following.count).to eq 2
+        expect(account.following.count).to eq 1
+        expect(account.follow_requests.count).to eq 1
         expect(Follow.find_by(account: account, target_account: bob).show_reblogs).to be true
-        expect(Follow.find_by(account: account, target_account: eve).show_reblogs).to be false
+        expect(FollowRequest.find_by(account: account, target_account: eve).show_reblogs).to be false
       end
     end
 
@@ -148,9 +156,10 @@ RSpec.describe ImportService, type: :service do
       it 'mutes the listed accounts, respecting notifications' do
         account.follow!(bob, reblogs: true)
         subject.call(import)
-        expect(account.following.count).to eq 2
+        expect(account.following.count).to eq 1
+        expect(account.follow_requests.count).to eq 1
         expect(Follow.find_by(account: account, target_account: bob).show_reblogs).to be true
-        expect(Follow.find_by(account: account, target_account: eve).show_reblogs).to be false
+        expect(FollowRequest.find_by(account: account, target_account: eve).show_reblogs).to be false
       end
     end
 
@@ -160,9 +169,10 @@ RSpec.describe ImportService, type: :service do
       it 'mutes the listed accounts, respecting notifications' do
         account.follow!(bob, reblogs: true)
         subject.call(import)
-        expect(account.following.count).to eq 2
+        expect(account.following.count).to eq 1
+        expect(account.follow_requests.count).to eq 1
         expect(Follow.find_by(account: account, target_account: bob).show_reblogs).to be true
-        expect(Follow.find_by(account: account, target_account: eve).show_reblogs).to be false
+        expect(FollowRequest.find_by(account: account, target_account: eve).show_reblogs).to be false
       end
     end
   end
diff --git a/spec/services/post_status_service_spec.rb b/spec/services/post_status_service_spec.rb
index facbe977f..bf06f50e9 100644
--- a/spec/services/post_status_service_spec.rb
+++ b/spec/services/post_status_service_spec.rb
@@ -144,7 +144,6 @@ RSpec.describe PostStatusService, type: :service do
 
   it 'gets distributed' do
     allow(DistributionWorker).to receive(:perform_async)
-    allow(Pubsubhubbub::DistributionWorker).to receive(:perform_async)
     allow(ActivityPub::DistributionWorker).to receive(:perform_async)
 
     account = Fabricate(:account)
@@ -152,7 +151,6 @@ RSpec.describe PostStatusService, type: :service do
     status = subject.call(account, text: "test status update")
 
     expect(DistributionWorker).to have_received(:perform_async).with(status.id)
-    expect(Pubsubhubbub::DistributionWorker).to have_received(:perform_async).with(status.stream_entry.id)
     expect(ActivityPub::DistributionWorker).to have_received(:perform_async).with(status.id)
   end
 
diff --git a/spec/services/process_feed_service_spec.rb b/spec/services/process_feed_service_spec.rb
deleted file mode 100644
index 9d3465f3f..000000000
--- a/spec/services/process_feed_service_spec.rb
+++ /dev/null
@@ -1,252 +0,0 @@
-require 'rails_helper'
-
-RSpec.describe ProcessFeedService, type: :service do
-  subject { ProcessFeedService.new }
-
-  describe 'processing a feed' do
-    let(:body) { File.read(Rails.root.join('spec', 'fixtures', 'xml', 'mastodon.atom')) }
-    let(:account) { Fabricate(:account, username: 'localhost', domain: 'kickass.zone') }
-
-    before do
-      stub_request(:post, "https://pubsubhubbub.superfeedr.com/").to_return(:status => 200, :body => "", :headers => {})
-      stub_request(:head, "http://kickass.zone/media/2").to_return(:status => 404)
-      stub_request(:head, "http://kickass.zone/media/3").to_return(:status => 404)
-      stub_request(:get, "http://kickass.zone/system/accounts/avatars/000/000/001/large/eris.png").to_return(request_fixture('avatar.txt'))
-      stub_request(:get, "http://kickass.zone/system/media_attachments/files/000/000/002/original/morpheus_linux.jpg?1476059910").to_return(request_fixture('attachment1.txt'))
-      stub_request(:get, "http://kickass.zone/system/media_attachments/files/000/000/003/original/gizmo.jpg?1476060065").to_return(request_fixture('attachment2.txt'))
-    end
-
-    context 'when domain does not reject media' do
-      before do
-        subject.call(body, account)
-      end
-
-      it 'updates remote user\'s account information' do
-        account.reload
-        expect(account.display_name).to eq '::1'
-        expect(account).to have_attached_file(:avatar)
-        expect(account.avatar_file_name).not_to be_nil
-      end
-
-      it 'creates posts' do
-        expect(Status.find_by(uri: 'tag:kickass.zone,2016-10-10:objectId=1:objectType=Status')).to_not be_nil
-        expect(Status.find_by(uri: 'tag:kickass.zone,2016-10-10:objectId=2:objectType=Status')).to_not be_nil
-      end
-
-      it 'marks replies as replies' do
-        status = Status.find_by(uri: 'tag:kickass.zone,2016-10-10:objectId=2:objectType=Status')
-        expect(status.reply?).to be true
-      end
-
-      it 'sets account being replied to when possible' do
-        status = Status.find_by(uri: 'tag:kickass.zone,2016-10-10:objectId=2:objectType=Status')
-        expect(status.in_reply_to_account_id).to eq status.account_id
-      end
-
-      it 'ignores delete statuses unless they existed before' do
-        expect(Status.find_by(uri: 'tag:kickass.zone,2016-10-10:objectId=3:objectType=Status')).to be_nil
-        expect(Status.find_by(uri: 'tag:kickass.zone,2016-10-10:objectId=12:objectType=Status')).to be_nil
-      end
-
-      it 'does not create statuses for follows' do
-        expect(Status.find_by(uri: 'tag:kickass.zone,2016-10-10:objectId=1:objectType=Follow')).to be_nil
-        expect(Status.find_by(uri: 'tag:kickass.zone,2016-10-10:objectId=2:objectType=Follow')).to be_nil
-        expect(Status.find_by(uri: 'tag:kickass.zone,2016-10-10:objectId=4:objectType=Follow')).to be_nil
-        expect(Status.find_by(uri: 'tag:kickass.zone,2016-10-10:objectId=7:objectType=Follow')).to be_nil
-      end
-
-      it 'does not create statuses for favourites' do
-        expect(Status.find_by(uri: 'tag:kickass.zone,2016-10-10:objectId=2:objectType=Favourite')).to be_nil
-        expect(Status.find_by(uri: 'tag:kickass.zone,2016-10-10:objectId=3:objectType=Favourite')).to be_nil
-      end
-
-      it 'creates posts with media' do
-        status = Status.find_by(uri: 'tag:kickass.zone,2016-10-10:objectId=14:objectType=Status')
-
-        expect(status).to_not be_nil
-        expect(status.media_attachments.first).to have_attached_file(:file)
-        expect(status.media_attachments.first.image?).to be true
-        expect(status.media_attachments.first.file_file_name).not_to be_nil
-      end
-    end
-
-    context 'when domain is set to reject media' do
-      let!(:domain_block) { Fabricate(:domain_block, domain: 'kickass.zone', reject_media: true) }
-
-      before do
-        subject.call(body, account)
-      end
-
-      it 'updates remote user\'s account information' do
-        account.reload
-        expect(account.display_name).to eq '::1'
-      end
-
-      it 'rejects remote user\'s avatar' do
-        account.reload
-        expect(account.display_name).to eq '::1'
-        expect(account.avatar_file_name).to be_nil
-      end
-
-      it 'creates posts' do
-        expect(Status.find_by(uri: 'tag:kickass.zone,2016-10-10:objectId=1:objectType=Status')).to_not be_nil
-        expect(Status.find_by(uri: 'tag:kickass.zone,2016-10-10:objectId=2:objectType=Status')).to_not be_nil
-      end
-
-      it 'creates posts with remote-only media' do
-        status = Status.find_by(uri: 'tag:kickass.zone,2016-10-10:objectId=14:objectType=Status')
-
-        expect(status).to_not be_nil
-        expect(status.media_attachments.first.file_file_name).to be_nil
-        expect(status.media_attachments.first.unknown?).to be true
-      end
-    end
-  end
-
-  it 'does not accept tampered reblogs' do
-    good_actor = Fabricate(:account, username: 'tracer', domain: 'overwatch.com')
-
-    real_body = <<XML
-<?xml version="1.0"?>
-<entry xmlns="http://www.w3.org/2005/Atom" xmlns:thr="http://purl.org/syndication/thread/1.0" xmlns:activity="http://activitystrea.ms/spec/1.0/" xmlns:poco="http://portablecontacts.net/spec/1.0" xmlns:media="http://purl.org/syndication/atommedia" xmlns:ostatus="http://ostatus.org/schema/1.0" xmlns:mastodon="http://mastodon.social/schema/1.0">
-  <id>tag:overwatch.com,2017-04-27:objectId=4467137:objectType=Status</id>
-  <published>2017-04-27T13:49:25Z</published>
-  <updated>2017-04-27T13:49:25Z</updated>
-  <activity:object-type>http://activitystrea.ms/schema/1.0/note</activity:object-type>
-  <activity:verb>http://activitystrea.ms/schema/1.0/post</activity:verb>
-  <author>
-    <id>https://overwatch.com/users/tracer</id>
-    <activity:object-type>http://activitystrea.ms/schema/1.0/person</activity:object-type>
-    <uri>https://overwatch.com/users/tracer</uri>
-    <name>tracer</name>
-  </author>
-  <content type="html">Overwatch rocks</content>
-</entry>
-XML
-
-    stub_request(:get, 'https://overwatch.com/users/tracer/updates/1').to_return(status: 200, body: real_body, headers: { 'Content-Type' => 'application/atom+xml' })
-
-    bad_actor = Fabricate(:account, username: 'sombra', domain: 'talon.xyz')
-
-    body = <<XML
-<?xml version="1.0"?>
-<entry xmlns="http://www.w3.org/2005/Atom" xmlns:thr="http://purl.org/syndication/thread/1.0" xmlns:activity="http://activitystrea.ms/spec/1.0/" xmlns:poco="http://portablecontacts.net/spec/1.0" xmlns:media="http://purl.org/syndication/atommedia" xmlns:ostatus="http://ostatus.org/schema/1.0" xmlns:mastodon="http://mastodon.social/schema/1.0">
-  <id>tag:talon.xyz,2017-04-27:objectId=4467137:objectType=Status</id>
-  <published>2017-04-27T13:49:25Z</published>
-  <updated>2017-04-27T13:49:25Z</updated>
-  <author>
-    <id>https://talon.xyz/users/sombra</id>
-    <activity:object-type>http://activitystrea.ms/schema/1.0/person</activity:object-type>
-    <uri>https://talon.xyz/users/sombra</uri>
-    <name>sombra</name>
-  </author>
-  <activity:object-type>http://activitystrea.ms/schema/1.0/activity</activity:object-type>
-  <activity:verb>http://activitystrea.ms/schema/1.0/share</activity:verb>
-  <content type="html">Overwatch SUCKS AHAHA</content>
-  <activity:object>
-    <id>tag:overwatch.com,2017-04-27:objectId=4467137:objectType=Status</id>
-    <activity:object-type>http://activitystrea.ms/schema/1.0/note</activity:object-type>
-    <activity:verb>http://activitystrea.ms/schema/1.0/post</activity:verb>
-    <author>
-      <id>https://overwatch.com/users/tracer</id>
-      <activity:object-type>http://activitystrea.ms/schema/1.0/person</activity:object-type>
-      <uri>https://overwatch.com/users/tracer</uri>
-      <name>tracer</name>
-    </author>
-    <content type="html">Overwatch SUCKS AHAHA</content>
-    <link rel="alternate" type="text/html" href="https://overwatch.com/users/tracer/updates/1" />
-  </activity:object>
-</entry>
-XML
-    created_statuses = subject.call(body, bad_actor)
-
-    expect(created_statuses.first.reblog?).to be true
-    expect(created_statuses.first.account_id).to eq bad_actor.id
-    expect(created_statuses.first.reblog.account_id).to eq good_actor.id
-    expect(created_statuses.first.reblog.text).to eq 'Overwatch rocks'
-  end
-
-  it 'ignores reblogs if it failed to retrieve reblogged statuses' do
-    stub_request(:get, 'https://overwatch.com/users/tracer/updates/1').to_return(status: 404)
-
-    actor = Fabricate(:account, username: 'tracer', domain: 'overwatch.com')
-
-    body = <<XML
-<?xml version="1.0"?>
-<entry xmlns="http://www.w3.org/2005/Atom" xmlns:thr="http://purl.org/syndication/thread/1.0" xmlns:activity="http://activitystrea.ms/spec/1.0/" xmlns:poco="http://portablecontacts.net/spec/1.0" xmlns:media="http://purl.org/syndication/atommedia" xmlns:ostatus="http://ostatus.org/schema/1.0" xmlns:mastodon="http://mastodon.social/schema/1.0">
-  <id>tag:overwatch.com,2017-04-27:objectId=4467137:objectType=Status</id>
-  <published>2017-04-27T13:49:25Z</published>
-  <updated>2017-04-27T13:49:25Z</updated>
-  <author>
-    <id>https://overwatch.com/users/tracer</id>
-    <activity:object-type>http://activitystrea.ms/schema/1.0/person</activity:object-type>
-    <uri>https://overwatch.com/users/tracer</uri>
-    <name>tracer</name>
-  </author>
-  <activity:object-type>http://activitystrea.ms/schema/1.0/activity</activity:object-type>
-  <activity:verb>http://activitystrea.ms/schema/1.0/share</activity:verb>
-  <content type="html">Overwatch rocks</content>
-  <activity:object>
-    <id>tag:overwatch.com,2017-04-27:objectId=4467137:objectType=Status</id>
-    <activity:object-type>http://activitystrea.ms/schema/1.0/note</activity:object-type>
-    <activity:verb>http://activitystrea.ms/schema/1.0/post</activity:verb>
-    <author>
-      <id>https://overwatch.com/users/tracer</id>
-      <activity:object-type>http://activitystrea.ms/schema/1.0/person</activity:object-type>
-      <uri>https://overwatch.com/users/tracer</uri>
-      <name>tracer</name>
-    </author>
-    <content type="html">Overwatch rocks</content>
-    <link rel="alternate" type="text/html" href="https://overwatch.com/users/tracer/updates/1" />
-  </activity:object>
-XML
-
-    created_statuses = subject.call(body, actor)
-
-    expect(created_statuses).to eq []
-  end
-
-  it 'ignores statuses with an out-of-order delete' do
-    sender = Fabricate(:account, username: 'tracer', domain: 'overwatch.com')
-
-    delete_body = <<XML
-<?xml version="1.0"?>
-<entry xmlns="http://www.w3.org/2005/Atom" xmlns:thr="http://purl.org/syndication/thread/1.0" xmlns:activity="http://activitystrea.ms/spec/1.0/" xmlns:poco="http://portablecontacts.net/spec/1.0" xmlns:media="http://purl.org/syndication/atommedia" xmlns:ostatus="http://ostatus.org/schema/1.0" xmlns:mastodon="http://mastodon.social/schema/1.0">
-  <id>tag:overwatch.com,2017-04-27:objectId=4487555:objectType=Status</id>
-  <published>2017-04-27T13:49:25Z</published>
-  <updated>2017-04-27T13:49:25Z</updated>
-  <activity:object-type>http://activitystrea.ms/schema/1.0/note</activity:object-type>
-  <activity:verb>http://activitystrea.ms/schema/1.0/delete</activity:verb>
-  <author>
-    <id>https://overwatch.com/users/tracer</id>
-    <activity:object-type>http://activitystrea.ms/schema/1.0/person</activity:object-type>
-    <uri>https://overwatch.com/users/tracer</uri>
-    <name>tracer</name>
-  </author>
-</entry>
-XML
-
-    status_body = <<XML
-<?xml version="1.0"?>
-<entry xmlns="http://www.w3.org/2005/Atom" xmlns:thr="http://purl.org/syndication/thread/1.0" xmlns:activity="http://activitystrea.ms/spec/1.0/" xmlns:poco="http://portablecontacts.net/spec/1.0" xmlns:media="http://purl.org/syndication/atommedia" xmlns:ostatus="http://ostatus.org/schema/1.0" xmlns:mastodon="http://mastodon.social/schema/1.0">
-  <id>tag:overwatch.com,2017-04-27:objectId=4487555:objectType=Status</id>
-  <published>2017-04-27T13:49:25Z</published>
-  <updated>2017-04-27T13:49:25Z</updated>
-  <activity:object-type>http://activitystrea.ms/schema/1.0/note</activity:object-type>
-  <activity:verb>http://activitystrea.ms/schema/1.0/post</activity:verb>
-  <author>
-    <id>https://overwatch.com/users/tracer</id>
-    <activity:object-type>http://activitystrea.ms/schema/1.0/person</activity:object-type>
-    <uri>https://overwatch.com/users/tracer</uri>
-    <name>tracer</name>
-  </author>
-  <content type="html">Overwatch rocks</content>
-</entry>
-XML
-
-    subject.call(delete_body, sender)
-    created_statuses = subject.call(status_body, sender)
-
-    expect(created_statuses).to be_empty
-  end
-end
diff --git a/spec/services/process_interaction_service_spec.rb b/spec/services/process_interaction_service_spec.rb
deleted file mode 100644
index b858c19d0..000000000
--- a/spec/services/process_interaction_service_spec.rb
+++ /dev/null
@@ -1,151 +0,0 @@
-require 'rails_helper'
-
-RSpec.describe ProcessInteractionService, type: :service do
-  let(:receiver) { Fabricate(:user, email: 'alice@example.com', account: Fabricate(:account, username: 'alice')).account }
-  let(:sender)   { Fabricate(:user, email: 'bob@example.com', account: Fabricate(:account, username: 'bob')).account }
-  let(:remote_sender) { Fabricate(:account, username: 'carol', domain: 'localdomain.com', uri: 'https://webdomain.com/users/carol') }
-
-  subject { ProcessInteractionService.new }
-
-  describe 'status delete slap' do
-    let(:remote_status) { Fabricate(:status, account: remote_sender) }
-    let(:envelope) { OStatus2::Salmon.new.pack(payload, sender.keypair) }
-    let(:payload) {
-      <<~XML
-        <entry xmlns="http://www.w3.org/2005/Atom" xmlns:activity="http://activitystrea.ms/spec/1.0/">
-          <author>
-            <email>carol@localdomain.com</email>
-            <name>carol</name>
-            <uri>https://webdomain.com/users/carol</uri>
-          </author>
-
-          <id>#{remote_status.id}</id>
-          <activity:verb>http://activitystrea.ms/schema/1.0/delete</activity:verb>
-        </entry>
-      XML
-    }
-
-    before do
-      receiver.update(locked: true)
-      remote_sender.update(private_key: sender.private_key, public_key: remote_sender.public_key)
-    end
-
-    it 'deletes a record' do
-      expect(RemovalWorker).to receive(:perform_async).with(remote_status.id)
-      subject.call(envelope, receiver)
-    end
-  end
-
-  describe 'follow request slap' do
-    before do
-      receiver.update(locked: true)
-
-      payload = <<XML
-<entry xmlns="http://www.w3.org/2005/Atom" xmlns:activity="http://activitystrea.ms/spec/1.0/">
-  <author>
-    <name>bob</name>
-    <uri>https://cb6e6126.ngrok.io/users/bob</uri>
-  </author>
-
-  <id>someIdHere</id>
-  <activity:verb>http://activitystrea.ms/schema/1.0/request-friend</activity:verb>
-</entry>
-XML
-
-      envelope = OStatus2::Salmon.new.pack(payload, sender.keypair)
-      subject.call(envelope, receiver)
-    end
-
-    it 'creates a record' do
-      expect(FollowRequest.find_by(account: sender, target_account: receiver)).to_not be_nil
-    end
-  end
-
-  describe 'follow request slap from known remote user identified by email' do
-    before do
-      receiver.update(locked: true)
-      # Copy already-generated key
-      remote_sender.update(private_key: sender.private_key, public_key: remote_sender.public_key)
-
-      payload = <<XML
-<entry xmlns="http://www.w3.org/2005/Atom" xmlns:activity="http://activitystrea.ms/spec/1.0/">
-  <author>
-    <email>carol@localdomain.com</email>
-    <name>carol</name>
-    <uri>https://webdomain.com/users/carol</uri>
-  </author>
-
-  <id>someIdHere</id>
-  <activity:verb>http://activitystrea.ms/schema/1.0/request-friend</activity:verb>
-</entry>
-XML
-
-      envelope = OStatus2::Salmon.new.pack(payload, remote_sender.keypair)
-      subject.call(envelope, receiver)
-    end
-
-    it 'creates a record' do
-      expect(FollowRequest.find_by(account: remote_sender, target_account: receiver)).to_not be_nil
-    end
-  end
-
-  describe 'follow request authorization slap' do
-    before do
-      receiver.update(locked: true)
-      FollowRequest.create(account: sender, target_account: receiver)
-
-      payload = <<XML
-<entry xmlns="http://www.w3.org/2005/Atom" xmlns:activity="http://activitystrea.ms/spec/1.0/">
-  <author>
-    <name>alice</name>
-    <uri>https://cb6e6126.ngrok.io/users/alice</uri>
-  </author>
-
-  <id>someIdHere</id>
-  <activity:verb>http://activitystrea.ms/schema/1.0/authorize</activity:verb>
-</entry>
-XML
-
-      envelope = OStatus2::Salmon.new.pack(payload, receiver.keypair)
-      subject.call(envelope, sender)
-    end
-
-    it 'creates a follow relationship' do
-      expect(Follow.find_by(account: sender, target_account: receiver)).to_not be_nil
-    end
-
-    it 'removes the follow request' do
-      expect(FollowRequest.find_by(account: sender, target_account: receiver)).to be_nil
-    end
-  end
-
-  describe 'follow request rejection slap' do
-    before do
-      receiver.update(locked: true)
-      FollowRequest.create(account: sender, target_account: receiver)
-
-      payload = <<XML
-<entry xmlns="http://www.w3.org/2005/Atom" xmlns:activity="http://activitystrea.ms/spec/1.0/">
-  <author>
-    <name>alice</name>
-    <uri>https://cb6e6126.ngrok.io/users/alice</uri>
-  </author>
-
-  <id>someIdHere</id>
-  <activity:verb>http://activitystrea.ms/schema/1.0/reject</activity:verb>
-</entry>
-XML
-
-      envelope = OStatus2::Salmon.new.pack(payload, receiver.keypair)
-      subject.call(envelope, sender)
-    end
-
-    it 'does not create a follow relationship' do
-      expect(Follow.find_by(account: sender, target_account: receiver)).to be_nil
-    end
-
-    it 'removes the follow request' do
-      expect(FollowRequest.find_by(account: sender, target_account: receiver)).to be_nil
-    end
-  end
-end
diff --git a/spec/services/process_mentions_service_spec.rb b/spec/services/process_mentions_service_spec.rb
index 8a6bb44ac..35a804f2b 100644
--- a/spec/services/process_mentions_service_spec.rb
+++ b/spec/services/process_mentions_service_spec.rb
@@ -18,10 +18,6 @@ RSpec.describe ProcessMentionsService, type: :service do
     it 'creates a mention' do
       expect(remote_user.mentions.where(status: status).count).to eq 1
     end
-
-    it 'posts to remote user\'s Salmon end point' do
-      expect(a_request(:post, remote_user.salmon_url)).to have_been_made.once
-    end
   end
 
   context 'OStatus with private toot' do
diff --git a/spec/services/pubsubhubbub/subscribe_service_spec.rb b/spec/services/pubsubhubbub/subscribe_service_spec.rb
deleted file mode 100644
index 01c956230..000000000
--- a/spec/services/pubsubhubbub/subscribe_service_spec.rb
+++ /dev/null
@@ -1,71 +0,0 @@
-# frozen_string_literal: true
-
-require 'rails_helper'
-
-describe Pubsubhubbub::SubscribeService, type: :service do
-  describe '#call' do
-    subject { described_class.new }
-    let(:user_account) { Fabricate(:account) }
-
-    context 'with a nil account' do
-      it 'returns the invalid topic status results' do
-        result = service_call(account: nil)
-
-        expect(result).to eq invalid_topic_status
-      end
-    end
-
-    context 'with an invalid callback url' do
-      it 'returns invalid callback status when callback is blank' do
-        result = service_call(callback: '')
-
-        expect(result).to eq invalid_callback_status
-      end
-      it 'returns invalid callback status when callback is not a URI' do
-        result = service_call(callback: 'invalid-hostname')
-
-        expect(result).to eq invalid_callback_status
-      end
-    end
-
-    context 'with a blocked domain in the callback' do
-      it 'returns callback not allowed' do
-        Fabricate(:domain_block, domain: 'test.host', severity: :suspend)
-        result = service_call(callback: 'https://test.host/api')
-
-        expect(result).to eq not_allowed_callback_status
-      end
-    end
-
-    context 'with a valid account and callback' do
-      it 'returns success status and confirms subscription' do
-        allow(Pubsubhubbub::ConfirmationWorker).to receive(:perform_async).and_return(nil)
-        subscription = Fabricate(:subscription, account: user_account)
-
-        result = service_call(callback: subscription.callback_url)
-        expect(result).to eq success_status
-        expect(Pubsubhubbub::ConfirmationWorker).to have_received(:perform_async).with(subscription.id, 'subscribe', 'asdf', 3600)
-      end
-    end
-  end
-
-  def service_call(account: user_account, callback: 'https://callback.host', secret: 'asdf', lease_seconds: 3600)
-    subject.call(account, callback, secret, lease_seconds)
-  end
-
-  def invalid_topic_status
-    ['Invalid topic URL', 422]
-  end
-
-  def invalid_callback_status
-    ['Invalid callback URL', 422]
-  end
-
-  def not_allowed_callback_status
-    ['Callback URL not allowed', 403]
-  end
-
-  def success_status
-    ['', 202]
-  end
-end
diff --git a/spec/services/pubsubhubbub/unsubscribe_service_spec.rb b/spec/services/pubsubhubbub/unsubscribe_service_spec.rb
deleted file mode 100644
index 7ed9fc5af..000000000
--- a/spec/services/pubsubhubbub/unsubscribe_service_spec.rb
+++ /dev/null
@@ -1,46 +0,0 @@
-# frozen_string_literal: true
-
-require 'rails_helper'
-
-describe Pubsubhubbub::UnsubscribeService, type: :service do
-  describe '#call' do
-    subject { described_class.new }
-
-    context 'with a nil account' do
-      it 'returns an invalid topic status' do
-        result = subject.call(nil, 'callback.host')
-
-        expect(result).to eq invalid_topic_status
-      end
-    end
-
-    context 'with a valid account' do
-      let(:account) { Fabricate(:account) }
-
-      it 'returns a valid topic status and does not run confirm when no subscription' do
-        allow(Pubsubhubbub::ConfirmationWorker).to receive(:perform_async).and_return(nil)
-        result = subject.call(account, 'callback.host')
-
-        expect(result).to eq valid_topic_status
-        expect(Pubsubhubbub::ConfirmationWorker).not_to have_received(:perform_async)
-      end
-
-      it 'returns a valid topic status and does run confirm when there is a subscription' do
-        subscription = Fabricate(:subscription, account: account, callback_url: 'callback.host')
-        allow(Pubsubhubbub::ConfirmationWorker).to receive(:perform_async).and_return(nil)
-        result = subject.call(account, 'callback.host')
-
-        expect(result).to eq valid_topic_status
-        expect(Pubsubhubbub::ConfirmationWorker).to have_received(:perform_async).with(subscription.id, 'unsubscribe')
-      end
-    end
-
-    def invalid_topic_status
-      ['Invalid topic URL', 422]
-    end
-
-    def valid_topic_status
-      ['', 202]
-    end
-  end
-end
diff --git a/spec/services/reblog_service_spec.rb b/spec/services/reblog_service_spec.rb
index 9d84c41d5..58fb46f0f 100644
--- a/spec/services/reblog_service_spec.rb
+++ b/spec/services/reblog_service_spec.rb
@@ -46,10 +46,6 @@ RSpec.describe ReblogService, type: :service do
     it 'creates a reblog' do
       expect(status.reblogs.count).to eq 1
     end
-
-    it 'sends a Salmon slap for a remote reblog' do
-      expect(a_request(:post, 'http://salmon.example.com')).to have_been_made
-    end
   end
 
   context 'ActivityPub' do
diff --git a/spec/services/reject_follow_service_spec.rb b/spec/services/reject_follow_service_spec.rb
index e5ac37ed9..1aec060db 100644
--- a/spec/services/reject_follow_service_spec.rb
+++ b/spec/services/reject_follow_service_spec.rb
@@ -38,13 +38,6 @@ RSpec.describe RejectFollowService, type: :service do
     it 'does not create follow relation' do
       expect(bob.following?(sender)).to be false
     end
-
-    it 'sends a follow request rejection salmon slap' do
-      expect(a_request(:post, "http://salmon.example.com/").with { |req|
-        xml = OStatus2::Salmon.new.unpack(req.body)
-        xml.match(OStatus::TagManager::VERBS[:reject])
-      }).to have_been_made.once
-    end
   end
 
   describe 'remote ActivityPub' do
diff --git a/spec/services/remove_status_service_spec.rb b/spec/services/remove_status_service_spec.rb
index 7bba83a60..48191d47c 100644
--- a/spec/services/remove_status_service_spec.rb
+++ b/spec/services/remove_status_service_spec.rb
@@ -32,23 +32,10 @@ RSpec.describe RemoveStatusService, type: :service do
     expect(HomeFeed.new(jeff).get(10)).to_not include(@status.id)
   end
 
-  it 'sends PuSH update to PuSH subscribers' do
-    expect(a_request(:post, 'http://example.com/push').with { |req|
-      req.body.match(OStatus::TagManager::VERBS[:delete])
-    }).to have_been_made
-  end
-
   it 'sends delete activity to followers' do
     expect(a_request(:post, 'http://example.com/inbox')).to have_been_made.twice
   end
 
-  it 'sends Salmon slap to previously mentioned users' do
-    expect(a_request(:post, "http://example.com/salmon").with { |req|
-      xml = OStatus2::Salmon.new.unpack(req.body)
-      xml.match(OStatus::TagManager::VERBS[:delete])
-    }).to have_been_made.once
-  end
-
   it 'sends delete activity to rebloggers' do
     expect(a_request(:post, 'http://example2.com/inbox')).to have_been_made
   end
diff --git a/spec/services/resolve_account_service_spec.rb b/spec/services/resolve_account_service_spec.rb
index 27a85af7c..7a64f4161 100644
--- a/spec/services/resolve_account_service_spec.rb
+++ b/spec/services/resolve_account_service_spec.rb
@@ -6,19 +6,13 @@ RSpec.describe ResolveAccountService, type: :service do
   before do
     stub_request(:get, "https://quitter.no/.well-known/host-meta").to_return(request_fixture('.host-meta.txt'))
     stub_request(:get, "https://example.com/.well-known/webfinger?resource=acct:catsrgr8@example.com").to_return(status: 404)
-    stub_request(:get, "https://redirected.com/.well-known/host-meta").to_return(request_fixture('redirected.host-meta.txt'))
     stub_request(:get, "https://example.com/.well-known/host-meta").to_return(status: 404)
-    stub_request(:get, "https://quitter.no/.well-known/webfinger?resource=acct:gargron@quitter.no").to_return(request_fixture('webfinger.txt'))
-    stub_request(:get, "https://redirected.com/.well-known/webfinger?resource=acct:gargron@redirected.com").to_return(request_fixture('webfinger.txt'))
-    stub_request(:get, "https://redirected.com/.well-known/webfinger?resource=acct:hacker1@redirected.com").to_return(request_fixture('webfinger-hacker1.txt'))
-    stub_request(:get, "https://redirected.com/.well-known/webfinger?resource=acct:hacker2@redirected.com").to_return(request_fixture('webfinger-hacker2.txt'))
-    stub_request(:get, "https://quitter.no/.well-known/webfinger?resource=acct:catsrgr8@quitter.no").to_return(status: 404)
-    stub_request(:get, "https://quitter.no/api/statuses/user_timeline/7477.atom").to_return(request_fixture('feed.txt'))
     stub_request(:get, "https://quitter.no/avatar/7477-300-20160211190340.png").to_return(request_fixture('avatar.txt'))
-    stub_request(:get, "https://localdomain.com/.well-known/host-meta").to_return(request_fixture('localdomain-hostmeta.txt'))
-    stub_request(:get, "https://localdomain.com/.well-known/webfinger?resource=acct:foo@localdomain.com").to_return(status: 404)
-    stub_request(:get, "https://webdomain.com/.well-known/webfinger?resource=acct:foo@localdomain.com").to_return(request_fixture('localdomain-webfinger.txt'))
-    stub_request(:get, "https://webdomain.com/users/foo.atom").to_return(request_fixture('localdomain-feed.txt'))
+    stub_request(:get, "https://quitter.no/.well-known/webfinger?resource=acct:catsrgr8@quitter.no").to_return(status: 404)
+    stub_request(:get, "https://ap.example.com/.well-known/webfinger?resource=acct:foo@ap.example.com").to_return(request_fixture('activitypub-webfinger.txt'))
+    stub_request(:get, "https://ap.example.com/users/foo").to_return(request_fixture('activitypub-actor.txt'))
+    stub_request(:get, "https://ap.example.com/users/foo.atom").to_return(request_fixture('activitypub-feed.txt'))
+    stub_request(:get, %r{https://ap.example.com/users/foo/\w+}).to_return(status: 404)
   end
 
   it 'raises error if no such user can be resolved via webfinger' do
@@ -29,74 +23,7 @@ RSpec.describe ResolveAccountService, type: :service do
     expect(subject.call('catsrgr8@example.com')).to be_nil
   end
 
-  it 'prevents hijacking existing accounts' do
-    account = subject.call('hacker1@redirected.com')
-    expect(account.salmon_url).to_not eq 'https://hacker.com/main/salmon/user/7477'
-  end
-
-  it 'prevents hijacking inexisting accounts' do
-    expect(subject.call('hacker2@redirected.com')).to be_nil
-  end
-
-  context 'with an OStatus account' do
-    it 'returns an already existing remote account' do
-      old_account      = Fabricate(:account, username: 'gargron', domain: 'quitter.no')
-      returned_account = subject.call('gargron@quitter.no')
-
-      expect(old_account.id).to eq returned_account.id
-    end
-
-    it 'returns a new remote account' do
-      account = subject.call('gargron@quitter.no')
-
-      expect(account.username).to eq 'gargron'
-      expect(account.domain).to eq 'quitter.no'
-      expect(account.remote_url).to eq 'https://quitter.no/api/statuses/user_timeline/7477.atom'
-    end
-
-    it 'follows a legitimate account redirection' do
-      account = subject.call('gargron@redirected.com')
-
-      expect(account.username).to eq 'gargron'
-      expect(account.domain).to eq 'quitter.no'
-      expect(account.remote_url).to eq 'https://quitter.no/api/statuses/user_timeline/7477.atom'
-    end
-
-    it 'returns a new remote account' do
-      account = subject.call('foo@localdomain.com')
-
-      expect(account.username).to eq 'foo'
-      expect(account.domain).to eq 'localdomain.com'
-      expect(account.remote_url).to eq 'https://webdomain.com/users/foo.atom'
-    end
-  end
-
   context 'with an ActivityPub account' do
-    before do
-      stub_request(:get, "https://ap.example.com/.well-known/webfinger?resource=acct:foo@ap.example.com").to_return(request_fixture('activitypub-webfinger.txt'))
-      stub_request(:get, "https://ap.example.com/users/foo").to_return(request_fixture('activitypub-actor.txt'))
-      stub_request(:get, "https://ap.example.com/users/foo.atom").to_return(request_fixture('activitypub-feed.txt'))
-      stub_request(:get, %r{https://ap.example.com/users/foo/\w+}).to_return(status: 404)
-    end
-
-    it 'fallback to OStatus if actor json could not be fetched' do
-      stub_request(:get, "https://ap.example.com/users/foo").to_return(status: 404)
-
-      account = subject.call('foo@ap.example.com')
-
-      expect(account.ostatus?).to eq true
-      expect(account.remote_url).to eq 'https://ap.example.com/users/foo.atom'
-    end
-
-    it 'fallback to OStatus if actor json did not have inbox_url' do
-      stub_request(:get, "https://ap.example.com/users/foo").to_return(request_fixture('activitypub-actor-noinbox.txt'))
-
-      account = subject.call('foo@ap.example.com')
-
-      expect(account.ostatus?).to eq true
-      expect(account.remote_url).to eq 'https://ap.example.com/users/foo.atom'
-    end
-
     it 'returns new remote account' do
       account = subject.call('foo@ap.example.com')
 
@@ -124,13 +51,14 @@ RSpec.describe ResolveAccountService, type: :service do
   it 'processes one remote account at a time using locks' do
     wait_for_start = true
     fail_occurred  = false
-    return_values  = []
+    return_values  = Concurrent::Array.new
 
     threads = Array.new(5) do
       Thread.new do
         true while wait_for_start
+
         begin
-          return_values << described_class.new.call('foo@localdomain.com')
+          return_values << described_class.new.call('foo@ap.example.com')
         rescue ActiveRecord::RecordNotUnique
           fail_occurred = true
         end
diff --git a/spec/services/send_interaction_service_spec.rb b/spec/services/send_interaction_service_spec.rb
deleted file mode 100644
index 710d8184c..000000000
--- a/spec/services/send_interaction_service_spec.rb
+++ /dev/null
@@ -1,7 +0,0 @@
-require 'rails_helper'
-
-RSpec.describe SendInteractionService, type: :service do
-  subject { SendInteractionService.new }
-
-  it 'sends an XML envelope to the Salmon end point of remote user'
-end
diff --git a/spec/services/subscribe_service_spec.rb b/spec/services/subscribe_service_spec.rb
deleted file mode 100644
index 10bdb1ba8..000000000
--- a/spec/services/subscribe_service_spec.rb
+++ /dev/null
@@ -1,43 +0,0 @@
-require 'rails_helper'
-
-RSpec.describe SubscribeService, type: :service do
-  let(:account) { Fabricate(:account, username: 'bob', domain: 'example.com', hub_url: 'http://hub.example.com') }
-  subject { SubscribeService.new }
-
-  it 'sends subscription request to PuSH hub' do
-    stub_request(:post, 'http://hub.example.com/').to_return(status: 202)
-    subject.call(account)
-    expect(a_request(:post, 'http://hub.example.com/')).to have_been_made.once
-  end
-
-  it 'generates and keeps PuSH secret on successful call' do
-    stub_request(:post, 'http://hub.example.com/').to_return(status: 202)
-    subject.call(account)
-    expect(account.secret).to_not be_blank
-  end
-
-  it 'fails silently if PuSH hub forbids subscription' do
-    stub_request(:post, 'http://hub.example.com/').to_return(status: 403)
-    subject.call(account)
-  end
-
-  it 'fails silently if PuSH hub is not found' do
-    stub_request(:post, 'http://hub.example.com/').to_return(status: 404)
-    subject.call(account)
-  end
-
-  it 'fails loudly if there is a network error' do
-    stub_request(:post, 'http://hub.example.com/').to_raise(HTTP::Error)
-    expect { subject.call(account) }.to raise_error HTTP::Error
-  end
-
-  it 'fails loudly if PuSH hub is unavailable' do
-    stub_request(:post, 'http://hub.example.com/').to_return(status: 503)
-    expect { subject.call(account) }.to raise_error Mastodon::UnexpectedResponseError
-  end
-
-  it 'fails loudly if rate limited' do
-    stub_request(:post, 'http://hub.example.com/').to_return(status: 429)
-    expect { subject.call(account) }.to raise_error Mastodon::UnexpectedResponseError
-  end
-end
diff --git a/spec/services/unblock_service_spec.rb b/spec/services/unblock_service_spec.rb
index 5835b912b..6350c6834 100644
--- a/spec/services/unblock_service_spec.rb
+++ b/spec/services/unblock_service_spec.rb
@@ -30,13 +30,6 @@ RSpec.describe UnblockService, type: :service do
     it 'destroys the blocking relation' do
       expect(sender.blocking?(bob)).to be false
     end
-
-    it 'sends an unblock salmon slap' do
-      expect(a_request(:post, "http://salmon.example.com/").with { |req|
-        xml = OStatus2::Salmon.new.unpack(req.body)
-        xml.match(OStatus::TagManager::VERBS[:unblock])
-      }).to have_been_made.once
-    end
   end
 
   describe 'remote ActivityPub' do
diff --git a/spec/services/unfollow_service_spec.rb b/spec/services/unfollow_service_spec.rb
index 8a2881ab1..84b5dafbc 100644
--- a/spec/services/unfollow_service_spec.rb
+++ b/spec/services/unfollow_service_spec.rb
@@ -30,13 +30,6 @@ RSpec.describe UnfollowService, type: :service do
     it 'destroys the following relation' do
       expect(sender.following?(bob)).to be false
     end
-
-    it 'sends an unfollow salmon slap' do
-      expect(a_request(:post, "http://salmon.example.com/").with { |req|
-        xml = OStatus2::Salmon.new.unpack(req.body)
-        xml.match(OStatus::TagManager::VERBS[:unfollow])
-      }).to have_been_made.once
-    end
   end
 
   describe 'remote ActivityPub' do
diff --git a/spec/services/unsubscribe_service_spec.rb b/spec/services/unsubscribe_service_spec.rb
deleted file mode 100644
index 54d4b1b53..000000000
--- a/spec/services/unsubscribe_service_spec.rb
+++ /dev/null
@@ -1,37 +0,0 @@
-require 'rails_helper'
-
-RSpec.describe UnsubscribeService, type: :service do
-  let(:account) { Fabricate(:account, username: 'bob', domain: 'example.com', hub_url: 'http://hub.example.com') }
-  subject { UnsubscribeService.new }
-
-  it 'removes the secret and resets expiration on account' do
-    stub_request(:post, 'http://hub.example.com/').to_return(status: 204)
-    subject.call(account)
-    account.reload
-
-    expect(account.secret).to be_blank
-    expect(account.subscription_expires_at).to be_blank
-  end
-
-  it 'logs error on subscription failure' do
-    logger = stub_logger
-    stub_request(:post, 'http://hub.example.com/').to_return(status: 404)
-    subject.call(account)
-
-    expect(logger).to have_received(:debug).with(/unsubscribe for bob@example.com failed/)
-  end
-
-  it 'logs error on connection failure' do
-    logger = stub_logger
-    stub_request(:post, 'http://hub.example.com/').to_raise(HTTP::Error)
-    subject.call(account)
-
-    expect(logger).to have_received(:debug).with(/unsubscribe for bob@example.com failed/)
-  end
-
-  def stub_logger
-    double(debug: nil).tap do |logger|
-      allow(Rails).to receive(:logger).and_return(logger)
-    end
-  end
-end
diff --git a/spec/services/update_remote_profile_service_spec.rb b/spec/services/update_remote_profile_service_spec.rb
deleted file mode 100644
index f3ea70b80..000000000
--- a/spec/services/update_remote_profile_service_spec.rb
+++ /dev/null
@@ -1,84 +0,0 @@
-require 'rails_helper'
-
-RSpec.describe UpdateRemoteProfileService, type: :service do
-  let(:xml) { File.read(Rails.root.join('spec', 'fixtures', 'push', 'feed.atom')) }
-
-  subject { UpdateRemoteProfileService.new }
-
-  before do
-    stub_request(:get, 'https://quitter.no/avatar/7477-300-20160211190340.png').to_return(request_fixture('avatar.txt'))
-  end
-
-  context 'with updated details' do
-    let(:remote_account) { Fabricate(:account, username: 'bob', domain: 'example.com') }
-
-    before do
-      subject.call(xml, remote_account)
-    end
-
-    it 'downloads new avatar' do
-      expect(a_request(:get, 'https://quitter.no/avatar/7477-300-20160211190340.png')).to have_been_made
-    end
-
-    it 'sets the avatar remote url' do
-      expect(remote_account.reload.avatar_remote_url).to eq 'https://quitter.no/avatar/7477-300-20160211190340.png'
-    end
-
-    it 'sets display name' do
-      expect(remote_account.reload.display_name).to eq 'ＤＩＧＩＴＡＬ ＣＡＴ'
-    end
-
-    it 'sets note' do
-      expect(remote_account.reload.note).to eq 'Software engineer, free time musician and ＤＩＧＩＴＡＬ ＳＰＯＲＴＳ enthusiast. Likes cats. Warning: May contain memes'
-    end
-  end
-
-  context 'with unchanged details' do
-    let(:remote_account) { Fabricate(:account, username: 'bob', domain: 'example.com', display_name: 'ＤＩＧＩＴＡＬ ＣＡＴ', note: 'Software engineer, free time musician and ＤＩＧＩＴＡＬ ＳＰＯＲＴＳ enthusiast. Likes cats. Warning: May contain memes', avatar_remote_url: 'https://quitter.no/avatar/7477-300-20160211190340.png') }
-
-    before do
-      subject.call(xml, remote_account)
-    end
-
-    it 'does not re-download avatar' do
-      expect(a_request(:get, 'https://quitter.no/avatar/7477-300-20160211190340.png')).to have_been_made.once
-    end
-
-    it 'sets the avatar remote url' do
-      expect(remote_account.reload.avatar_remote_url).to eq 'https://quitter.no/avatar/7477-300-20160211190340.png'
-    end
-
-    it 'sets display name' do
-      expect(remote_account.reload.display_name).to eq 'ＤＩＧＩＴＡＬ ＣＡＴ'
-    end
-
-    it 'sets note' do
-      expect(remote_account.reload.note).to eq 'Software engineer, free time musician and ＤＩＧＩＴＡＬ ＳＰＯＲＴＳ enthusiast. Likes cats. Warning: May contain memes'
-    end
-  end
-
-  context 'with updated details from a domain set to reject media' do
-    let(:remote_account) { Fabricate(:account, username: 'bob', domain: 'example.com') }
-    let!(:domain_block) { Fabricate(:domain_block, domain: 'example.com', reject_media: true) }
-
-    before do
-      subject.call(xml, remote_account)
-    end
-
-    it 'does not the avatar remote url' do
-      expect(remote_account.reload.avatar_remote_url).to be_nil
-    end
-
-    it 'sets display name' do
-      expect(remote_account.reload.display_name).to eq 'ＤＩＧＩＴＡＬ ＣＡＴ'
-    end
-
-    it 'sets note' do
-      expect(remote_account.reload.note).to eq 'Software engineer, free time musician and ＤＩＧＩＴＡＬ ＳＰＯＲＴＳ enthusiast. Likes cats. Warning: May contain memes'
-    end
-
-    it 'does not set store the avatar' do
-      expect(remote_account.reload.avatar_file_name).to be_nil
-    end
-  end
-end
diff --git a/spec/workers/after_remote_follow_request_worker_spec.rb b/spec/workers/after_remote_follow_request_worker_spec.rb
deleted file mode 100644
index bd623cca5..000000000
--- a/spec/workers/after_remote_follow_request_worker_spec.rb
+++ /dev/null
@@ -1,59 +0,0 @@
-# frozen_string_literal: true
-
-require 'rails_helper'
-
-describe AfterRemoteFollowRequestWorker do
-  subject { described_class.new }
-  let(:follow_request) { Fabricate(:follow_request) }
-  describe 'perform' do
-    context 'when the follow_request does not exist' do
-      it 'catches a raise and returns true' do
-        allow(FollowService).to receive(:new)
-        result = subject.perform('aaa')
-
-        expect(result).to eq(true)
-        expect(FollowService).not_to have_received(:new)
-      end
-    end
-
-    context 'when the account cannot be updated' do
-      it 'returns nil and does not call service when account is nil' do
-        allow(FollowService).to receive(:new)
-        service = double(call: nil)
-        allow(FetchRemoteAccountService).to receive(:new).and_return(service)
-
-        result = subject.perform(follow_request.id)
-
-        expect(result).to be_nil
-        expect(FollowService).not_to have_received(:new)
-      end
-
-      it 'returns nil and does not call service when account is locked' do
-        allow(FollowService).to receive(:new)
-        service = double(call: double(locked?: true))
-        allow(FetchRemoteAccountService).to receive(:new).and_return(service)
-
-        result = subject.perform(follow_request.id)
-
-        expect(result).to be_nil
-        expect(FollowService).not_to have_received(:new)
-      end
-    end
-
-    context 'when the account is updated' do
-      it 'calls the follow service and destroys the follow' do
-        follow_service = double(call: nil)
-        allow(FollowService).to receive(:new).and_return(follow_service)
-        account = Fabricate(:account, locked: false)
-        service = double(call: account)
-        allow(FetchRemoteAccountService).to receive(:new).and_return(service)
-
-        result = subject.perform(follow_request.id)
-
-        expect(result).to be_nil
-        expect(follow_service).to have_received(:call).with(follow_request.account, account.acct)
-        expect { follow_request.reload }.to raise_error(ActiveRecord::RecordNotFound)
-      end
-    end
-  end
-end
diff --git a/spec/workers/after_remote_follow_worker_spec.rb b/spec/workers/after_remote_follow_worker_spec.rb
deleted file mode 100644
index d93c469f9..000000000
--- a/spec/workers/after_remote_follow_worker_spec.rb
+++ /dev/null
@@ -1,59 +0,0 @@
-# frozen_string_literal: true
-
-require 'rails_helper'
-
-describe AfterRemoteFollowWorker do
-  subject { described_class.new }
-  let(:follow) { Fabricate(:follow) }
-  describe 'perform' do
-    context 'when the follow does not exist' do
-      it 'catches a raise and returns true' do
-        allow(FollowService).to receive(:new)
-        result = subject.perform('aaa')
-
-        expect(result).to eq(true)
-        expect(FollowService).not_to have_received(:new)
-      end
-    end
-
-    context 'when the account cannot be updated' do
-      it 'returns nil and does not call service when account is nil' do
-        allow(FollowService).to receive(:new)
-        service = double(call: nil)
-        allow(FetchRemoteAccountService).to receive(:new).and_return(service)
-
-        result = subject.perform(follow.id)
-
-        expect(result).to be_nil
-        expect(FollowService).not_to have_received(:new)
-      end
-
-      it 'returns nil and does not call service when account is not locked' do
-        allow(FollowService).to receive(:new)
-        service = double(call: double(locked?: false))
-        allow(FetchRemoteAccountService).to receive(:new).and_return(service)
-
-        result = subject.perform(follow.id)
-
-        expect(result).to be_nil
-        expect(FollowService).not_to have_received(:new)
-      end
-    end
-
-    context 'when the account is updated' do
-      it 'calls the follow service and destroys the follow' do
-        follow_service = double(call: nil)
-        allow(FollowService).to receive(:new).and_return(follow_service)
-        account = Fabricate(:account, locked: true)
-        service = double(call: account)
-        allow(FetchRemoteAccountService).to receive(:new).and_return(service)
-
-        result = subject.perform(follow.id)
-
-        expect(result).to be_nil
-        expect(follow_service).to have_received(:call).with(follow.account, account.acct)
-        expect { follow.reload }.to raise_error(ActiveRecord::RecordNotFound)
-      end
-    end
-  end
-end
diff --git a/spec/workers/pubsubhubbub/confirmation_worker_spec.rb b/spec/workers/pubsubhubbub/confirmation_worker_spec.rb
deleted file mode 100644
index 1eecdd2b5..000000000
--- a/spec/workers/pubsubhubbub/confirmation_worker_spec.rb
+++ /dev/null
@@ -1,88 +0,0 @@
-# frozen_string_literal: true
-
-require 'rails_helper'
-
-describe Pubsubhubbub::ConfirmationWorker do
-  include RoutingHelper
-
-  subject { described_class.new }
-
-  let!(:alice) { Fabricate(:account, username: 'alice') }
-  let!(:subscription) { Fabricate(:subscription, account: alice, callback_url: 'http://example.com/api', confirmed: false, expires_at: 3.days.from_now, secret: nil) }
-
-  describe 'perform' do
-    describe 'with subscribe mode' do
-      it 'confirms and updates subscription when challenge matches' do
-        stub_random_value
-        stub_request(:get, url_for_mode('subscribe'))
-          .with(headers: http_headers)
-          .to_return(status: 200, body: challenge_value, headers: {})
-
-        seconds = 10.days.seconds.to_i
-        subject.perform(subscription.id, 'subscribe', 'asdf', seconds)
-
-        subscription.reload
-        expect(subscription.secret).to eq 'asdf'
-        expect(subscription.confirmed).to eq true
-        expect(subscription.expires_at).to be_within(5).of(10.days.from_now)
-      end
-
-      it 'does not update subscription when challenge does not match' do
-        stub_random_value
-        stub_request(:get, url_for_mode('subscribe'))
-          .with(headers: http_headers)
-          .to_return(status: 200, body: 'wrong value', headers: {})
-
-        seconds = 10.days.seconds.to_i
-        subject.perform(subscription.id, 'subscribe', 'asdf', seconds)
-
-        subscription.reload
-        expect(subscription.secret).to be_blank
-        expect(subscription.confirmed).to eq false
-        expect(subscription.expires_at).to be_within(5).of(3.days.from_now)
-      end
-    end
-
-    describe 'with unsubscribe mode' do
-      it 'confirms and destroys subscription when challenge matches' do
-        stub_random_value
-        stub_request(:get, url_for_mode('unsubscribe'))
-          .with(headers: http_headers)
-          .to_return(status: 200, body: challenge_value, headers: {})
-
-        seconds = 10.days.seconds.to_i
-        subject.perform(subscription.id, 'unsubscribe', 'asdf', seconds)
-
-        expect { subscription.reload }.to raise_error(ActiveRecord::RecordNotFound)
-      end
-
-      it 'does not destroy subscription when challenge does not match' do
-        stub_random_value
-        stub_request(:get, url_for_mode('unsubscribe'))
-          .with(headers: http_headers)
-          .to_return(status: 200, body: 'wrong value', headers: {})
-
-        seconds = 10.days.seconds.to_i
-        subject.perform(subscription.id, 'unsubscribe', 'asdf', seconds)
-
-        expect { subscription.reload }.not_to raise_error
-      end
-    end
-  end
-
-  def url_for_mode(mode)
-    "http://example.com/api?hub.challenge=#{challenge_value}&hub.lease_seconds=863999&hub.mode=#{mode}&hub.topic=https://#{Rails.configuration.x.local_domain}/users/alice.atom"
-  end
-
-  def stub_random_value
-    allow(SecureRandom).to receive(:hex).and_return(challenge_value)
-  end
-
-  def challenge_value
-    '1a2s3d4f'
-  end
-
-  def http_headers
-    { 'Connection' => 'close', 'Host' => 'example.com' }
-  end
-end
diff --git a/spec/workers/pubsubhubbub/delivery_worker_spec.rb b/spec/workers/pubsubhubbub/delivery_worker_spec.rb
deleted file mode 100644
index c0e0d5186..000000000
--- a/spec/workers/pubsubhubbub/delivery_worker_spec.rb
+++ /dev/null
@@ -1,68 +0,0 @@
-# frozen_string_literal: true
-
-require 'rails_helper'
-
-describe Pubsubhubbub::DeliveryWorker do
-  include RoutingHelper
-  subject { described_class.new }
-
-  let(:payload) { 'test' }
-
-  describe 'perform' do
-    it 'raises when subscription does not exist' do
-      expect { subject.perform 123, payload }.to raise_error(ActiveRecord::RecordNotFound)
-    end
-
-    it 'does not attempt to deliver when domain blocked' do
-      _domain_block = Fabricate(:domain_block, domain: 'example.com', severity: :suspend)
-      subscription = Fabricate(:subscription, callback_url: 'https://example.com/api', last_successful_delivery_at: 2.days.ago)
-
-      subject.perform(subscription.id, payload)
-
-      expect(subscription.reload.last_successful_delivery_at).to be_within(2).of(2.days.ago)
-    end
-
-    it 'raises when request fails' do
-      subscription = Fabricate(:subscription)
-
-      stub_request_to_respond_with(subscription, 500)
-      expect { subject.perform(subscription.id, payload) }.to raise_error Mastodon::UnexpectedResponseError
-    end
-
-    it 'updates subscriptions when delivery succeeds' do
-      subscription = Fabricate(:subscription)
-
-      stub_request_to_respond_with(subscription, 200)
-      subject.perform(subscription.id, payload)
-
-      expect(subscription.reload.last_successful_delivery_at).to be_within(2).of(Time.now.utc)
-    end
-
-    it 'updates subscription without a secret when delivery succeeds' do
-      subscription = Fabricate(:subscription, secret: nil)
-
-      stub_request_to_respond_with(subscription, 200)
-      subject.perform(subscription.id, payload)
-
-      expect(subscription.reload.last_successful_delivery_at).to be_within(2).of(Time.now.utc)
-    end
-
-    def stub_request_to_respond_with(subscription, code)
-      stub_request(:post, 'http://example.com/callback')
-        .with(body: payload, headers: expected_headers(subscription))
-        .to_return(status: code, body: '', headers: {})
-    end
-
-    def expected_headers(subscription)
-      {
-        'Connection' => 'close',
-        'Content-Type' => 'application/atom+xml',
-        'Host' => 'example.com',
-        'Link' => "<https://#{Rails.configuration.x.local_domain}/api/push>; rel=\"hub\", <https://#{Rails.configuration.x.local_domain}/users/#{subscription.account.username}.atom>; rel=\"self\"",
-      }.tap do |basic|
-        known_digest = OpenSSL::HMAC.hexdigest(OpenSSL::Digest.new('sha1'), subscription.secret.to_s, payload)
-        basic.merge('X-Hub-Signature' => "sha1=#{known_digest}") if subscription.secret?
-      end
-    end
-  end
-end
diff --git a/spec/workers/pubsubhubbub/distribution_worker_spec.rb b/spec/workers/pubsubhubbub/distribution_worker_spec.rb
deleted file mode 100644
index 584485079..000000000
--- a/spec/workers/pubsubhubbub/distribution_worker_spec.rb
+++ /dev/null
@@ -1,46 +0,0 @@
-require 'rails_helper'
-
-describe Pubsubhubbub::DistributionWorker do
-  subject { Pubsubhubbub::DistributionWorker.new }
-
-  let!(:alice) { Fabricate(:account, username: 'alice') }
-  let!(:bob) { Fabricate(:account, username: 'bob', domain: 'example2.com') }
-  let!(:anonymous_subscription) { Fabricate(:subscription, account: alice, callback_url: 'http://example1.com', confirmed: true, lease_seconds: 3600) }
-  let!(:subscription_with_follower) { Fabricate(:subscription, account: alice, callback_url: 'http://example2.com', confirmed: true, lease_seconds: 3600) }
-
-  before do
-    bob.follow!(alice)
-  end
-
-  describe 'with public status' do
-    let(:status) { Fabricate(:status, account: alice, text: 'Hello', visibility: :public) }
-
-    it 'delivers payload to all subscriptions' do
-      allow(Pubsubhubbub::DeliveryWorker).to receive(:push_bulk)
-      subject.perform(status.stream_entry.id)
-      expect(Pubsubhubbub::DeliveryWorker).to have_received(:push_bulk).with([anonymous_subscription.id, subscription_with_follower.id])
-    end
-  end
-
-  context 'when OStatus privacy is not used' do
-    describe 'with private status' do
-      let(:status) { Fabricate(:status, account: alice, text: 'Hello', visibility: :private) }
-
-      it 'does not deliver anything' do
-        allow(Pubsubhubbub::DeliveryWorker).to receive(:push_bulk)
-        subject.perform(status.stream_entry.id)
-        expect(Pubsubhubbub::DeliveryWorker).to_not have_received(:push_bulk)
-      end
-    end
-
-    describe 'with direct status' do
-      let(:status) { Fabricate(:status, account: alice, text: 'Hello', visibility: :direct) }
-
-      it 'does not deliver payload' do
-        allow(Pubsubhubbub::DeliveryWorker).to receive(:push_bulk)
-        subject.perform(status.stream_entry.id)
-        expect(Pubsubhubbub::DeliveryWorker).to_not have_received(:push_bulk)
-      end
-    end
-  end
-end
diff --git a/spec/workers/scheduler/subscriptions_scheduler_spec.rb b/spec/workers/scheduler/subscriptions_scheduler_spec.rb
deleted file mode 100644
index a7d1046de..000000000
--- a/spec/workers/scheduler/subscriptions_scheduler_spec.rb
+++ /dev/null
@@ -1,19 +0,0 @@
-require 'rails_helper'
-
-describe Scheduler::SubscriptionsScheduler do
-  subject { Scheduler::SubscriptionsScheduler.new }
-
-  let!(:expiring_account1) { Fabricate(:account, subscription_expires_at: 20.minutes.from_now, domain: 'example.com', followers_count: 1, hub_url: 'http://hub.example.com') }
-  let!(:expiring_account2) { Fabricate(:account, subscription_expires_at: 4.hours.from_now, domain: 'example.org', followers_count: 1, hub_url: 'http://hub.example.org') }
-
-  before do
-    stub_request(:post, 'http://hub.example.com/').to_return(status: 202)
-    stub_request(:post, 'http://hub.example.org/').to_return(status: 202)
-  end
-
-  it 're-subscribes for all expiring accounts' do
-    subject.perform
-    expect(a_request(:post, 'http://hub.example.com/')).to have_been_made.once
-    expect(a_request(:post, 'http://hub.example.org/')).to have_been_made.once
-  end
-end
-- 
cgit 


From b8514561394767a10d3cf40132ada24d938c1680 Mon Sep 17 00:00:00 2001
From: Eugen Rochko <eugen@zeonfederated.com>
Date: Sun, 7 Jul 2019 16:16:51 +0200
Subject: Remove Atom feeds and old URLs in the form of `GET
 /:username/updates/:id` (#11247)

---
 app/controllers/accounts_controller.rb             |    7 -
 app/controllers/statuses_controller.rb             |   36 +-
 app/controllers/stream_entries_controller.rb       |   64 -
 app/helpers/admin/action_logs_helper.rb            |    2 +-
 app/helpers/home_helper.rb                         |    2 +-
 app/helpers/statuses_helper.rb                     |  222 +++
 app/helpers/stream_entries_helper.rb               |  220 ---
 app/javascript/styles/application.scss             |    2 +-
 app/javascript/styles/mastodon/statuses.scss       |  163 +++
 app/javascript/styles/mastodon/stream_entries.scss |  163 ---
 app/lib/formatter.rb                               |    2 +-
 app/lib/ostatus/atom_serializer.rb                 |  376 ------
 app/lib/status_finder.rb                           |    2 -
 app/lib/tag_manager.rb                             |   11 -
 app/mailers/admin_mailer.rb                        |    2 +-
 app/mailers/notification_mailer.rb                 |    2 +-
 app/models/concerns/account_associations.rb        |    1 -
 app/models/concerns/streamable.rb                  |   43 -
 app/models/remote_profile.rb                       |   57 -
 app/models/status.rb                               |    4 -
 app/models/stream_entry.rb                         |   59 -
 app/serializers/rest/account_serializer.rb         |    2 +-
 app/serializers/rest/status_serializer.rb          |    6 +-
 app/serializers/rss/account_serializer.rb          |    6 +-
 app/serializers/rss/tag_serializer.rb              |    4 +-
 app/services/batched_remove_status_service.rb      |    2 +-
 app/services/fetch_link_card_service.rb            |    2 +-
 app/services/process_mentions_service.rb           |    2 +-
 app/services/remove_status_service.rb              |   15 +-
 app/services/resolve_url_service.rb                |    5 +-
 app/services/suspend_account_service.rb            |    1 -
 app/views/accounts/_moved.html.haml                |    4 +-
 app/views/accounts/show.html.haml                  |    4 +-
 app/views/admin/accounts/_account.html.haml        |    2 +-
 app/views/admin/reports/_status.html.haml          |    2 +-
 app/views/application/_card.html.haml              |    2 +-
 .../_post_follow_actions.html.haml                 |    2 +-
 app/views/remote_interaction/new.html.haml         |    2 +-
 app/views/remote_unfollows/_card.html.haml         |    2 +-
 .../_post_follow_actions.html.haml                 |    2 +-
 app/views/statuses/_attachment_list.html.haml      |    8 +
 app/views/statuses/_detailed_status.html.haml      |   79 ++
 app/views/statuses/_og_description.html.haml       |    4 +
 app/views/statuses/_og_image.html.haml             |   38 +
 app/views/statuses/_poll.html.haml                 |   27 +
 app/views/statuses/_simple_status.html.haml        |   60 +
 app/views/statuses/_status.html.haml               |   62 +
 app/views/statuses/embed.html.haml                 |    3 +
 app/views/statuses/show.html.haml                  |   24 +
 .../stream_entries/_attachment_list.html.haml      |    8 -
 .../stream_entries/_detailed_status.html.haml      |   79 --
 app/views/stream_entries/_og_description.html.haml |    4 -
 app/views/stream_entries/_og_image.html.haml       |   38 -
 app/views/stream_entries/_poll.html.haml           |   27 -
 app/views/stream_entries/_simple_status.html.haml  |   60 -
 app/views/stream_entries/_status.html.haml         |   62 -
 app/views/stream_entries/embed.html.haml           |    3 -
 app/views/stream_entries/show.html.haml            |   25 -
 config/routes.rb                                   |    6 -
 .../20190706233204_drop_stream_entries.rb          |   13 +
 db/schema.rb                                       |   14 +-
 spec/controllers/accounts_controller_spec.rb       |   31 -
 spec/controllers/api/oembed_controller_spec.rb     |    2 +-
 spec/controllers/application_controller_spec.rb    |    4 -
 spec/controllers/statuses_controller_spec.rb       |   16 +-
 spec/controllers/stream_entries_controller_spec.rb |   95 --
 spec/fabricators/stream_entry_fabricator.rb        |    5 -
 .../admin/account_moderation_notes_helper_spec.rb  |    2 +-
 spec/helpers/statuses_helper_spec.rb               |  224 ++++
 spec/helpers/stream_entries_helper_spec.rb         |  224 ----
 spec/lib/activitypub/tag_manager_spec.rb           |    6 -
 spec/lib/ostatus/atom_serializer_spec.rb           | 1415 --------------------
 spec/lib/status_finder_spec.rb                     |    9 -
 spec/lib/tag_manager_spec.rb                       |   42 -
 spec/models/concerns/streamable_spec.rb            |   63 -
 spec/models/remote_profile_spec.rb                 |  143 --
 spec/models/stream_entry_spec.rb                   |  192 ---
 spec/services/process_mentions_service_spec.rb     |    4 +-
 spec/services/suspend_account_service_spec.rb      |    6 +-
 spec/views/statuses/show.html.haml_spec.rb         |   81 ++
 spec/views/stream_entries/show.html.haml_spec.rb   |   88 --
 81 files changed, 1071 insertions(+), 3732 deletions(-)
 delete mode 100644 app/controllers/stream_entries_controller.rb
 create mode 100644 app/helpers/statuses_helper.rb
 delete mode 100644 app/helpers/stream_entries_helper.rb
 create mode 100644 app/javascript/styles/mastodon/statuses.scss
 delete mode 100644 app/javascript/styles/mastodon/stream_entries.scss
 delete mode 100644 app/lib/ostatus/atom_serializer.rb
 delete mode 100644 app/models/concerns/streamable.rb
 delete mode 100644 app/models/remote_profile.rb
 delete mode 100644 app/models/stream_entry.rb
 create mode 100644 app/views/statuses/_attachment_list.html.haml
 create mode 100644 app/views/statuses/_detailed_status.html.haml
 create mode 100644 app/views/statuses/_og_description.html.haml
 create mode 100644 app/views/statuses/_og_image.html.haml
 create mode 100644 app/views/statuses/_poll.html.haml
 create mode 100644 app/views/statuses/_simple_status.html.haml
 create mode 100644 app/views/statuses/_status.html.haml
 create mode 100644 app/views/statuses/embed.html.haml
 create mode 100644 app/views/statuses/show.html.haml
 delete mode 100644 app/views/stream_entries/_attachment_list.html.haml
 delete mode 100644 app/views/stream_entries/_detailed_status.html.haml
 delete mode 100644 app/views/stream_entries/_og_description.html.haml
 delete mode 100644 app/views/stream_entries/_og_image.html.haml
 delete mode 100644 app/views/stream_entries/_poll.html.haml
 delete mode 100644 app/views/stream_entries/_simple_status.html.haml
 delete mode 100644 app/views/stream_entries/_status.html.haml
 delete mode 100644 app/views/stream_entries/embed.html.haml
 delete mode 100644 app/views/stream_entries/show.html.haml
 create mode 100644 db/post_migrate/20190706233204_drop_stream_entries.rb
 delete mode 100644 spec/controllers/stream_entries_controller_spec.rb
 delete mode 100644 spec/fabricators/stream_entry_fabricator.rb
 create mode 100644 spec/helpers/statuses_helper_spec.rb
 delete mode 100644 spec/helpers/stream_entries_helper_spec.rb
 delete mode 100644 spec/lib/ostatus/atom_serializer_spec.rb
 delete mode 100644 spec/models/concerns/streamable_spec.rb
 delete mode 100644 spec/models/remote_profile_spec.rb
 delete mode 100644 spec/models/stream_entry_spec.rb
 create mode 100644 spec/views/statuses/show.html.haml_spec.rb
 delete mode 100644 spec/views/stream_entries/show.html.haml_spec.rb

(limited to 'spec')

diff --git a/app/controllers/accounts_controller.rb b/app/controllers/accounts_controller.rb
index 73a4b1859..065707378 100644
--- a/app/controllers/accounts_controller.rb
+++ b/app/controllers/accounts_controller.rb
@@ -31,13 +31,6 @@ class AccountsController < ApplicationController
         end
       end
 
-      format.atom do
-        mark_cacheable!
-
-        @entries = @account.stream_entries.where(hidden: false).with_includes.paginate_by_max_id(PAGE_SIZE, params[:max_id], params[:since_id])
-        render xml: OStatus::AtomSerializer.render(OStatus::AtomSerializer.new.feed(@account, @entries.reject { |entry| entry.status.nil? }))
-      end
-
       format.rss do
         mark_cacheable!
 
diff --git a/app/controllers/statuses_controller.rb b/app/controllers/statuses_controller.rb
index ef26691b2..776099ca8 100644
--- a/app/controllers/statuses_controller.rb
+++ b/app/controllers/statuses_controller.rb
@@ -33,12 +33,10 @@ class StatusesController < ApplicationController
 
         set_ancestors
         set_descendants
-
-        render 'stream_entries/show'
       end
 
       format.json do
-        render_cached_json(['activitypub', 'note', @status], content_type: 'application/activity+json', public: !@stream_entry.hidden?) do
+        render_cached_json(['activitypub', 'note', @status], content_type: 'application/activity+json', public: @status.distributable?) do
           ActiveModelSerializers::SerializableResource.new(@status, serializer: ActivityPub::NoteSerializer, adapter: ActivityPub::Adapter)
         end
       end
@@ -46,7 +44,7 @@ class StatusesController < ApplicationController
   end
 
   def activity
-    render_cached_json(['activitypub', 'activity', @status], content_type: 'application/activity+json', public: !@stream_entry.hidden?) do
+    render_cached_json(['activitypub', 'activity', @status], content_type: 'application/activity+json', public: @status.distributable?) do
       ActiveModelSerializers::SerializableResource.new(@status, serializer: ActivityPub::ActivitySerializer, adapter: ActivityPub::Adapter)
     end
   end
@@ -58,7 +56,7 @@ class StatusesController < ApplicationController
     response.headers['X-Frame-Options'] = 'ALLOWALL'
     @autoplay = ActiveModel::Type::Boolean.new.cast(params[:autoplay])
 
-    render 'stream_entries/embed', layout: 'embedded'
+    render layout: 'embedded'
   end
 
   def replies
@@ -92,11 +90,20 @@ class StatusesController < ApplicationController
 
   def create_descendant_thread(starting_depth, statuses)
     depth = starting_depth + statuses.size
+
     if depth < DESCENDANTS_DEPTH_LIMIT
-      { statuses: statuses, starting_depth: starting_depth }
+      {
+        statuses: statuses,
+        starting_depth: starting_depth,
+      }
     else
       next_status = statuses.pop
-      { statuses: statuses, starting_depth: starting_depth, next_status: next_status }
+
+      {
+        statuses: statuses,
+        starting_depth: starting_depth,
+        next_status: next_status,
+      }
     end
   end
 
@@ -164,22 +171,13 @@ class StatusesController < ApplicationController
   end
 
   def set_link_headers
-    response.headers['Link'] = LinkHeader.new(
-      [
-        [account_stream_entry_url(@account, @status.stream_entry, format: 'atom'), [%w(rel alternate), %w(type application/atom+xml)]],
-        [ActivityPub::TagManager.instance.uri_for(@status), [%w(rel alternate), %w(type application/activity+json)]],
-      ]
-    )
+    response.headers['Link'] = LinkHeader.new([[ActivityPub::TagManager.instance.uri_for(@status), [%w(rel alternate), %w(type application/activity+json)]]])
   end
 
   def set_status
-    @status       = @account.statuses.find(params[:id])
-    @stream_entry = @status.stream_entry
-    @type         = @stream_entry.activity_type.downcase
-
+    @status = @account.statuses.find(params[:id])
     authorize @status, :show?
   rescue Mastodon::NotPermittedError
-    # Reraise in order to get a 404
     raise ActiveRecord::RecordNotFound
   end
 
@@ -192,7 +190,7 @@ class StatusesController < ApplicationController
   end
 
   def redirect_to_original
-    redirect_to ::TagManager.instance.url_for(@status.reblog) if @status.reblog?
+    redirect_to ActivityPub::TagManager.instance.url_for(@status.reblog) if @status.reblog?
   end
 
   def set_referrer_policy_header
diff --git a/app/controllers/stream_entries_controller.rb b/app/controllers/stream_entries_controller.rb
deleted file mode 100644
index 0f7e9e0f5..000000000
--- a/app/controllers/stream_entries_controller.rb
+++ /dev/null
@@ -1,64 +0,0 @@
-# frozen_string_literal: true
-
-class StreamEntriesController < ApplicationController
-  include Authorization
-  include SignatureVerification
-
-  layout 'public'
-
-  before_action :set_account
-  before_action :set_stream_entry
-  before_action :set_link_headers
-  before_action :check_account_suspension
-  before_action :set_cache_headers
-
-  def show
-    respond_to do |format|
-      format.html do
-        expires_in 5.minutes, public: true unless @stream_entry.hidden?
-
-        redirect_to short_account_status_url(params[:account_username], @stream_entry.activity)
-      end
-
-      format.atom do
-        expires_in 3.minutes, public: true unless @stream_entry.hidden?
-
-        render xml: OStatus::AtomSerializer.render(OStatus::AtomSerializer.new.entry(@stream_entry, true))
-      end
-    end
-  end
-
-  def embed
-    redirect_to embed_short_account_status_url(@account, @stream_entry.activity), status: 301
-  end
-
-  private
-
-  def set_account
-    @account = Account.find_local!(params[:account_username])
-  end
-
-  def set_link_headers
-    response.headers['Link'] = LinkHeader.new(
-      [
-        [account_stream_entry_url(@account, @stream_entry, format: 'atom'), [%w(rel alternate), %w(type application/atom+xml)]],
-        [ActivityPub::TagManager.instance.uri_for(@stream_entry.activity), [%w(rel alternate), %w(type application/activity+json)]],
-      ]
-    )
-  end
-
-  def set_stream_entry
-    @stream_entry = @account.stream_entries.where(activity_type: 'Status').find(params[:id])
-    @type         = 'status'
-
-    raise ActiveRecord::RecordNotFound if @stream_entry.activity.nil?
-    authorize @stream_entry.activity, :show? if @stream_entry.hidden?
-  rescue Mastodon::NotPermittedError
-    # Reraise in order to get a 404
-    raise ActiveRecord::RecordNotFound
-  end
-
-  def check_account_suspension
-    gone if @account.suspended?
-  end
-end
diff --git a/app/helpers/admin/action_logs_helper.rb b/app/helpers/admin/action_logs_helper.rb
index e5fbb1500..1daa60774 100644
--- a/app/helpers/admin/action_logs_helper.rb
+++ b/app/helpers/admin/action_logs_helper.rb
@@ -89,7 +89,7 @@ module Admin::ActionLogsHelper
     when 'DomainBlock', 'EmailDomainBlock'
       link_to record.domain, "https://#{record.domain}"
     when 'Status'
-      link_to record.account.acct, TagManager.instance.url_for(record)
+      link_to record.account.acct, ActivityPub::TagManager.instance.url_for(record)
     when 'AccountWarning'
       link_to record.target_account.acct, admin_account_path(record.target_account_id)
     end
diff --git a/app/helpers/home_helper.rb b/app/helpers/home_helper.rb
index df60b7dd7..b66e827fe 100644
--- a/app/helpers/home_helper.rb
+++ b/app/helpers/home_helper.rb
@@ -21,7 +21,7 @@ module HomeHelper
                         end
                     end
                   else
-                    link_to(path || TagManager.instance.url_for(account), class: 'account__display-name') do
+                    link_to(path || ActivityPub::TagManager.instance.url_for(account), class: 'account__display-name') do
                       content_tag(:div, class: 'account__avatar-wrapper') do
                         content_tag(:div, '', class: 'account__avatar', style: "width: #{size}px; height: #{size}px; background-size: #{size}px #{size}px; background-image: url(#{full_asset_url(current_account&.user&.setting_auto_play_gif ? account.avatar_original_url : account.avatar_static_url)})")
                       end +
diff --git a/app/helpers/statuses_helper.rb b/app/helpers/statuses_helper.rb
new file mode 100644
index 000000000..e067380f6
--- /dev/null
+++ b/app/helpers/statuses_helper.rb
@@ -0,0 +1,222 @@
+# frozen_string_literal: true
+
+module StatusesHelper
+  EMBEDDED_CONTROLLER = 'statuses'
+  EMBEDDED_ACTION = 'embed'
+
+  def display_name(account, **options)
+    if options[:custom_emojify]
+      Formatter.instance.format_display_name(account, options)
+    else
+      account.display_name.presence || account.username
+    end
+  end
+
+  def account_action_button(account)
+    if user_signed_in?
+      if account.id == current_user.account_id
+        link_to settings_profile_url, class: 'button logo-button' do
+          safe_join([svg_logo, t('settings.edit_profile')])
+        end
+      elsif current_account.following?(account) || current_account.requested?(account)
+        link_to account_unfollow_path(account), class: 'button logo-button button--destructive', data: { method: :post } do
+          safe_join([svg_logo, t('accounts.unfollow')])
+        end
+      elsif !(account.memorial? || account.moved?)
+        link_to account_follow_path(account), class: "button logo-button#{account.blocking?(current_account) ? ' disabled' : ''}", data: { method: :post } do
+          safe_join([svg_logo, t('accounts.follow')])
+        end
+      end
+    elsif !(account.memorial? || account.moved?)
+      link_to account_remote_follow_path(account), class: 'button logo-button modal-button', target: '_new' do
+        safe_join([svg_logo, t('accounts.follow')])
+      end
+    end
+  end
+
+  def svg_logo
+    content_tag(:svg, tag(:use, 'xlink:href' => '#mastodon-svg-logo'), 'viewBox' => '0 0 216.4144 232.00976')
+  end
+
+  def svg_logo_full
+    content_tag(:svg, tag(:use, 'xlink:href' => '#mastodon-svg-logo-full'), 'viewBox' => '0 0 713.35878 175.8678')
+  end
+
+  def account_badge(account, all: false)
+    if account.bot?
+      content_tag(:div, content_tag(:div, t('accounts.roles.bot'), class: 'account-role bot'), class: 'roles')
+    elsif (Setting.show_staff_badge && account.user_staff?) || all
+      content_tag(:div, class: 'roles') do
+        if all && !account.user_staff?
+          content_tag(:div, t('admin.accounts.roles.user'), class: 'account-role')
+        elsif account.user_admin?
+          content_tag(:div, t('accounts.roles.admin'), class: 'account-role admin')
+        elsif account.user_moderator?
+          content_tag(:div, t('accounts.roles.moderator'), class: 'account-role moderator')
+        end
+      end
+    end
+  end
+
+  def link_to_more(url)
+    link_to t('statuses.show_more'), url, class: 'load-more load-gap'
+  end
+
+  def nothing_here(extra_classes = '')
+    content_tag(:div, class: "nothing-here #{extra_classes}") do
+      t('accounts.nothing_here')
+    end
+  end
+
+  def account_description(account)
+    prepend_str = [
+      [
+        number_to_human(account.statuses_count, strip_insignificant_zeros: true),
+        I18n.t('accounts.posts', count: account.statuses_count),
+      ].join(' '),
+
+      [
+        number_to_human(account.following_count, strip_insignificant_zeros: true),
+        I18n.t('accounts.following', count: account.following_count),
+      ].join(' '),
+
+      [
+        number_to_human(account.followers_count, strip_insignificant_zeros: true),
+        I18n.t('accounts.followers', count: account.followers_count),
+      ].join(' '),
+    ].join(', ')
+
+    [prepend_str, account.note].join(' · ')
+  end
+
+  def media_summary(status)
+    attachments = { image: 0, video: 0 }
+
+    status.media_attachments.each do |media|
+      if media.video?
+        attachments[:video] += 1
+      else
+        attachments[:image] += 1
+      end
+    end
+
+    text = attachments.to_a.reject { |_, value| value.zero? }.map { |key, value| I18n.t("statuses.attached.#{key}", count: value) }.join(' · ')
+
+    return if text.blank?
+
+    I18n.t('statuses.attached.description', attached: text)
+  end
+
+  def status_text_summary(status)
+    return if status.spoiler_text.blank?
+
+    I18n.t('statuses.content_warning', warning: status.spoiler_text)
+  end
+
+  def poll_summary(status)
+    return unless status.preloadable_poll
+
+    status.preloadable_poll.options.map { |o| "[ ] #{o}" }.join("\n")
+  end
+
+  def status_description(status)
+    components = [[media_summary(status), status_text_summary(status)].reject(&:blank?).join(' · ')]
+
+    if status.spoiler_text.blank?
+      components << status.text
+      components << poll_summary(status)
+    end
+
+    components.reject(&:blank?).join("\n\n")
+  end
+
+  def stream_link_target
+    embedded_view? ? '_blank' : nil
+  end
+
+  def acct(account)
+    if account.local?
+      "@#{account.acct}@#{Rails.configuration.x.local_domain}"
+    else
+      "@#{account.acct}"
+    end
+  end
+
+  def style_classes(status, is_predecessor, is_successor, include_threads)
+    classes = ['entry']
+    classes << 'entry-predecessor' if is_predecessor
+    classes << 'entry-reblog' if status.reblog?
+    classes << 'entry-successor' if is_successor
+    classes << 'entry-center' if include_threads
+    classes.join(' ')
+  end
+
+  def microformats_classes(status, is_direct_parent, is_direct_child)
+    classes = []
+    classes << 'p-in-reply-to' if is_direct_parent
+    classes << 'p-repost-of' if status.reblog? && is_direct_parent
+    classes << 'p-comment' if is_direct_child
+    classes.join(' ')
+  end
+
+  def microformats_h_class(status, is_predecessor, is_successor, include_threads)
+    if is_predecessor || status.reblog? || is_successor
+      'h-cite'
+    elsif include_threads
+      ''
+    else
+      'h-entry'
+    end
+  end
+
+  def rtl_status?(status)
+    status.local? ? rtl?(status.text) : rtl?(strip_tags(status.text))
+  end
+
+  def rtl?(text)
+    text = simplified_text(text)
+    rtl_words = text.scan(/[\p{Hebrew}\p{Arabic}\p{Syriac}\p{Thaana}\p{Nko}]+/m)
+
+    if rtl_words.present?
+      total_size = text.size.to_f
+      rtl_size(rtl_words) / total_size > 0.3
+    else
+      false
+    end
+  end
+
+  def fa_visibility_icon(status)
+    case status.visibility
+    when 'public'
+      fa_icon 'globe fw'
+    when 'unlisted'
+      fa_icon 'unlock fw'
+    when 'private'
+      fa_icon 'lock fw'
+    when 'direct'
+      fa_icon 'envelope fw'
+    end
+  end
+
+  private
+
+  def simplified_text(text)
+    text.dup.tap do |new_text|
+      URI.extract(new_text).each do |url|
+        new_text.gsub!(url, '')
+      end
+
+      new_text.gsub!(Account::MENTION_RE, '')
+      new_text.gsub!(Tag::HASHTAG_RE, '')
+      new_text.gsub!(/\s+/, '')
+    end
+  end
+
+  def rtl_size(words)
+    words.reduce(0) { |acc, elem| acc + elem.size }.to_f
+  end
+
+  def embedded_view?
+    params[:controller] == EMBEDDED_CONTROLLER && params[:action] == EMBEDDED_ACTION
+  end
+end
diff --git a/app/helpers/stream_entries_helper.rb b/app/helpers/stream_entries_helper.rb
deleted file mode 100644
index 02a860a74..000000000
--- a/app/helpers/stream_entries_helper.rb
+++ /dev/null
@@ -1,220 +0,0 @@
-# frozen_string_literal: true
-
-module StreamEntriesHelper
-  EMBEDDED_CONTROLLER = 'statuses'
-  EMBEDDED_ACTION = 'embed'
-
-  def display_name(account, **options)
-    if options[:custom_emojify]
-      Formatter.instance.format_display_name(account, options)
-    else
-      account.display_name.presence || account.username
-    end
-  end
-
-  def account_action_button(account)
-    if user_signed_in?
-      if account.id == current_user.account_id
-        link_to settings_profile_url, class: 'button logo-button' do
-          safe_join([svg_logo, t('settings.edit_profile')])
-        end
-      elsif current_account.following?(account) || current_account.requested?(account)
-        link_to account_unfollow_path(account), class: 'button logo-button button--destructive', data: { method: :post } do
-          safe_join([svg_logo, t('accounts.unfollow')])
-        end
-      elsif !(account.memorial? || account.moved?)
-        link_to account_follow_path(account), class: "button logo-button#{account.blocking?(current_account) ? ' disabled' : ''}", data: { method: :post } do
-          safe_join([svg_logo, t('accounts.follow')])
-        end
-      end
-    elsif !(account.memorial? || account.moved?)
-      link_to account_remote_follow_path(account), class: 'button logo-button modal-button', target: '_new' do
-        safe_join([svg_logo, t('accounts.follow')])
-      end
-    end
-  end
-
-  def svg_logo
-    content_tag(:svg, tag(:use, 'xlink:href' => '#mastodon-svg-logo'), 'viewBox' => '0 0 216.4144 232.00976')
-  end
-
-  def svg_logo_full
-    content_tag(:svg, tag(:use, 'xlink:href' => '#mastodon-svg-logo-full'), 'viewBox' => '0 0 713.35878 175.8678')
-  end
-
-  def account_badge(account, all: false)
-    if account.bot?
-      content_tag(:div, content_tag(:div, t('accounts.roles.bot'), class: 'account-role bot'), class: 'roles')
-    elsif (Setting.show_staff_badge && account.user_staff?) || all
-      content_tag(:div, class: 'roles') do
-        if all && !account.user_staff?
-          content_tag(:div, t('admin.accounts.roles.user'), class: 'account-role')
-        elsif account.user_admin?
-          content_tag(:div, t('accounts.roles.admin'), class: 'account-role admin')
-        elsif account.user_moderator?
-          content_tag(:div, t('accounts.roles.moderator'), class: 'account-role moderator')
-        end
-      end
-    end
-  end
-
-  def link_to_more(url)
-    link_to t('statuses.show_more'), url, class: 'load-more load-gap'
-  end
-
-  def nothing_here(extra_classes = '')
-    content_tag(:div, class: "nothing-here #{extra_classes}") do
-      t('accounts.nothing_here')
-    end
-  end
-
-  def account_description(account)
-    prepend_str = [
-      [
-        number_to_human(account.statuses_count, strip_insignificant_zeros: true),
-        I18n.t('accounts.posts', count: account.statuses_count),
-      ].join(' '),
-
-      [
-        number_to_human(account.following_count, strip_insignificant_zeros: true),
-        I18n.t('accounts.following', count: account.following_count),
-      ].join(' '),
-
-      [
-        number_to_human(account.followers_count, strip_insignificant_zeros: true),
-        I18n.t('accounts.followers', count: account.followers_count),
-      ].join(' '),
-    ].join(', ')
-
-    [prepend_str, account.note].join(' · ')
-  end
-
-  def media_summary(status)
-    attachments = { image: 0, video: 0 }
-
-    status.media_attachments.each do |media|
-      if media.video?
-        attachments[:video] += 1
-      else
-        attachments[:image] += 1
-      end
-    end
-
-    text = attachments.to_a.reject { |_, value| value.zero? }.map { |key, value| I18n.t("statuses.attached.#{key}", count: value) }.join(' · ')
-
-    return if text.blank?
-
-    I18n.t('statuses.attached.description', attached: text)
-  end
-
-  def status_text_summary(status)
-    return if status.spoiler_text.blank?
-    I18n.t('statuses.content_warning', warning: status.spoiler_text)
-  end
-
-  def poll_summary(status)
-    return unless status.preloadable_poll
-    status.preloadable_poll.options.map { |o| "[ ] #{o}" }.join("\n")
-  end
-
-  def status_description(status)
-    components = [[media_summary(status), status_text_summary(status)].reject(&:blank?).join(' · ')]
-
-    if status.spoiler_text.blank?
-      components << status.text
-      components << poll_summary(status)
-    end
-
-    components.reject(&:blank?).join("\n\n")
-  end
-
-  def stream_link_target
-    embedded_view? ? '_blank' : nil
-  end
-
-  def acct(account)
-    if account.local?
-      "@#{account.acct}@#{Rails.configuration.x.local_domain}"
-    else
-      "@#{account.acct}"
-    end
-  end
-
-  def style_classes(status, is_predecessor, is_successor, include_threads)
-    classes = ['entry']
-    classes << 'entry-predecessor' if is_predecessor
-    classes << 'entry-reblog' if status.reblog?
-    classes << 'entry-successor' if is_successor
-    classes << 'entry-center' if include_threads
-    classes.join(' ')
-  end
-
-  def microformats_classes(status, is_direct_parent, is_direct_child)
-    classes = []
-    classes << 'p-in-reply-to' if is_direct_parent
-    classes << 'p-repost-of' if status.reblog? && is_direct_parent
-    classes << 'p-comment' if is_direct_child
-    classes.join(' ')
-  end
-
-  def microformats_h_class(status, is_predecessor, is_successor, include_threads)
-    if is_predecessor || status.reblog? || is_successor
-      'h-cite'
-    elsif include_threads
-      ''
-    else
-      'h-entry'
-    end
-  end
-
-  def rtl_status?(status)
-    status.local? ? rtl?(status.text) : rtl?(strip_tags(status.text))
-  end
-
-  def rtl?(text)
-    text = simplified_text(text)
-    rtl_words = text.scan(/[\p{Hebrew}\p{Arabic}\p{Syriac}\p{Thaana}\p{Nko}]+/m)
-
-    if rtl_words.present?
-      total_size = text.size.to_f
-      rtl_size(rtl_words) / total_size > 0.3
-    else
-      false
-    end
-  end
-
-  def fa_visibility_icon(status)
-    case status.visibility
-    when 'public'
-      fa_icon 'globe fw'
-    when 'unlisted'
-      fa_icon 'unlock fw'
-    when 'private'
-      fa_icon 'lock fw'
-    when 'direct'
-      fa_icon 'envelope fw'
-    end
-  end
-
-  private
-
-  def simplified_text(text)
-    text.dup.tap do |new_text|
-      URI.extract(new_text).each do |url|
-        new_text.gsub!(url, '')
-      end
-
-      new_text.gsub!(Account::MENTION_RE, '')
-      new_text.gsub!(Tag::HASHTAG_RE, '')
-      new_text.gsub!(/\s+/, '')
-    end
-  end
-
-  def rtl_size(words)
-    words.reduce(0) { |acc, elem| acc + elem.size }.to_f
-  end
-
-  def embedded_view?
-    params[:controller] == EMBEDDED_CONTROLLER && params[:action] == EMBEDDED_ACTION
-  end
-end
diff --git a/app/javascript/styles/application.scss b/app/javascript/styles/application.scss
index 6db3bc3dc..8ebc45b62 100644
--- a/app/javascript/styles/application.scss
+++ b/app/javascript/styles/application.scss
@@ -13,7 +13,7 @@
 @import 'mastodon/widgets';
 @import 'mastodon/forms';
 @import 'mastodon/accounts';
-@import 'mastodon/stream_entries';
+@import 'mastodon/statuses';
 @import 'mastodon/boost';
 @import 'mastodon/components';
 @import 'mastodon/polls';
diff --git a/app/javascript/styles/mastodon/statuses.scss b/app/javascript/styles/mastodon/statuses.scss
new file mode 100644
index 000000000..19ce0ab8f
--- /dev/null
+++ b/app/javascript/styles/mastodon/statuses.scss
@@ -0,0 +1,163 @@
+.activity-stream {
+  box-shadow: 0 0 15px rgba($base-shadow-color, 0.2);
+  border-radius: 4px;
+  overflow: hidden;
+  margin-bottom: 10px;
+
+  &--under-tabs {
+    border-radius: 0 0 4px 4px;
+  }
+
+  @media screen and (max-width: $no-gap-breakpoint) {
+    margin-bottom: 0;
+    border-radius: 0;
+    box-shadow: none;
+  }
+
+  &--headless {
+    border-radius: 0;
+    margin: 0;
+    box-shadow: none;
+
+    .detailed-status,
+    .status {
+      border-radius: 0 !important;
+    }
+  }
+
+  div[data-component] {
+    width: 100%;
+  }
+
+  .entry {
+    background: $ui-base-color;
+
+    .detailed-status,
+    .status,
+    .load-more {
+      animation: none;
+    }
+
+    &:last-child {
+      .detailed-status,
+      .status,
+      .load-more {
+        border-bottom: 0;
+        border-radius: 0 0 4px 4px;
+      }
+    }
+
+    &:first-child {
+      .detailed-status,
+      .status,
+      .load-more {
+        border-radius: 4px 4px 0 0;
+      }
+
+      &:last-child {
+        .detailed-status,
+        .status,
+        .load-more {
+          border-radius: 4px;
+        }
+      }
+    }
+
+    @media screen and (max-width: 740px) {
+      .detailed-status,
+      .status,
+      .load-more {
+        border-radius: 0 !important;
+      }
+    }
+  }
+
+  &--highlighted .entry {
+    background: lighten($ui-base-color, 8%);
+  }
+}
+
+.button.logo-button {
+  flex: 0 auto;
+  font-size: 14px;
+  background: $ui-highlight-color;
+  color: $primary-text-color;
+  text-transform: none;
+  line-height: 36px;
+  height: auto;
+  padding: 3px 15px;
+  border: 0;
+
+  svg {
+    width: 20px;
+    height: auto;
+    vertical-align: middle;
+    margin-right: 5px;
+    fill: $primary-text-color;
+  }
+
+  &:active,
+  &:focus,
+  &:hover {
+    background: lighten($ui-highlight-color, 10%);
+  }
+
+  &:disabled,
+  &.disabled {
+    &:active,
+    &:focus,
+    &:hover {
+      background: $ui-primary-color;
+    }
+  }
+
+  &.button--destructive {
+    &:active,
+    &:focus,
+    &:hover {
+      background: $error-red;
+    }
+  }
+
+  @media screen and (max-width: $no-gap-breakpoint) {
+    svg {
+      display: none;
+    }
+  }
+}
+
+.embed,
+.public-layout {
+  .detailed-status {
+    padding: 15px;
+  }
+
+  .status {
+    padding: 15px 15px 15px (48px + 15px * 2);
+    min-height: 48px + 2px;
+
+    &__avatar {
+      left: 15px;
+      top: 17px;
+    }
+
+    &__content {
+      padding-top: 5px;
+    }
+
+    &__prepend {
+      margin-left: 48px + 15px * 2;
+      padding-top: 15px;
+    }
+
+    &__prepend-icon-wrapper {
+      left: -32px;
+    }
+
+    .media-gallery,
+    &__action-bar,
+    .video-player {
+      margin-top: 10px;
+    }
+  }
+}
diff --git a/app/javascript/styles/mastodon/stream_entries.scss b/app/javascript/styles/mastodon/stream_entries.scss
deleted file mode 100644
index 19ce0ab8f..000000000
--- a/app/javascript/styles/mastodon/stream_entries.scss
+++ /dev/null
@@ -1,163 +0,0 @@
-.activity-stream {
-  box-shadow: 0 0 15px rgba($base-shadow-color, 0.2);
-  border-radius: 4px;
-  overflow: hidden;
-  margin-bottom: 10px;
-
-  &--under-tabs {
-    border-radius: 0 0 4px 4px;
-  }
-
-  @media screen and (max-width: $no-gap-breakpoint) {
-    margin-bottom: 0;
-    border-radius: 0;
-    box-shadow: none;
-  }
-
-  &--headless {
-    border-radius: 0;
-    margin: 0;
-    box-shadow: none;
-
-    .detailed-status,
-    .status {
-      border-radius: 0 !important;
-    }
-  }
-
-  div[data-component] {
-    width: 100%;
-  }
-
-  .entry {
-    background: $ui-base-color;
-
-    .detailed-status,
-    .status,
-    .load-more {
-      animation: none;
-    }
-
-    &:last-child {
-      .detailed-status,
-      .status,
-      .load-more {
-        border-bottom: 0;
-        border-radius: 0 0 4px 4px;
-      }
-    }
-
-    &:first-child {
-      .detailed-status,
-      .status,
-      .load-more {
-        border-radius: 4px 4px 0 0;
-      }
-
-      &:last-child {
-        .detailed-status,
-        .status,
-        .load-more {
-          border-radius: 4px;
-        }
-      }
-    }
-
-    @media screen and (max-width: 740px) {
-      .detailed-status,
-      .status,
-      .load-more {
-        border-radius: 0 !important;
-      }
-    }
-  }
-
-  &--highlighted .entry {
-    background: lighten($ui-base-color, 8%);
-  }
-}
-
-.button.logo-button {
-  flex: 0 auto;
-  font-size: 14px;
-  background: $ui-highlight-color;
-  color: $primary-text-color;
-  text-transform: none;
-  line-height: 36px;
-  height: auto;
-  padding: 3px 15px;
-  border: 0;
-
-  svg {
-    width: 20px;
-    height: auto;
-    vertical-align: middle;
-    margin-right: 5px;
-    fill: $primary-text-color;
-  }
-
-  &:active,
-  &:focus,
-  &:hover {
-    background: lighten($ui-highlight-color, 10%);
-  }
-
-  &:disabled,
-  &.disabled {
-    &:active,
-    &:focus,
-    &:hover {
-      background: $ui-primary-color;
-    }
-  }
-
-  &.button--destructive {
-    &:active,
-    &:focus,
-    &:hover {
-      background: $error-red;
-    }
-  }
-
-  @media screen and (max-width: $no-gap-breakpoint) {
-    svg {
-      display: none;
-    }
-  }
-}
-
-.embed,
-.public-layout {
-  .detailed-status {
-    padding: 15px;
-  }
-
-  .status {
-    padding: 15px 15px 15px (48px + 15px * 2);
-    min-height: 48px + 2px;
-
-    &__avatar {
-      left: 15px;
-      top: 17px;
-    }
-
-    &__content {
-      padding-top: 5px;
-    }
-
-    &__prepend {
-      margin-left: 48px + 15px * 2;
-      padding-top: 15px;
-    }
-
-    &__prepend-icon-wrapper {
-      left: -32px;
-    }
-
-    .media-gallery,
-    &__action-bar,
-    .video-player {
-      margin-top: 10px;
-    }
-  }
-}
diff --git a/app/lib/formatter.rb b/app/lib/formatter.rb
index 8a1aad41a..6c1239963 100644
--- a/app/lib/formatter.rb
+++ b/app/lib/formatter.rb
@@ -295,6 +295,6 @@ class Formatter
   end
 
   def mention_html(account)
-    "<span class=\"h-card\"><a href=\"#{encode(TagManager.instance.url_for(account))}\" class=\"u-url mention\">@<span>#{encode(account.username)}</span></a></span>"
+    "<span class=\"h-card\"><a href=\"#{encode(ActivityPub::TagManager.instance.url_for(account))}\" class=\"u-url mention\">@<span>#{encode(account.username)}</span></a></span>"
   end
 end
diff --git a/app/lib/ostatus/atom_serializer.rb b/app/lib/ostatus/atom_serializer.rb
deleted file mode 100644
index f5c0e85ca..000000000
--- a/app/lib/ostatus/atom_serializer.rb
+++ /dev/null
@@ -1,376 +0,0 @@
-# frozen_string_literal: true
-
-class OStatus::AtomSerializer
-  include RoutingHelper
-  include ActionView::Helpers::SanitizeHelper
-
-  class << self
-    def render(element)
-      document = Ox::Document.new(version: '1.0')
-      document << element
-      ('<?xml version="1.0"?>' + Ox.dump(element, effort: :tolerant)).force_encoding('UTF-8')
-    end
-  end
-
-  def author(account)
-    author = Ox::Element.new('author')
-
-    uri = OStatus::TagManager.instance.uri_for(account)
-
-    append_element(author, 'id', uri)
-    append_element(author, 'activity:object-type', OStatus::TagManager::TYPES[:person])
-    append_element(author, 'uri', uri)
-    append_element(author, 'name', account.username)
-    append_element(author, 'email', account.local? ? account.local_username_and_domain : account.acct)
-    append_element(author, 'summary', Formatter.instance.simplified_format(account).to_str, type: :html) if account.note?
-    append_element(author, 'link', nil, rel: :alternate, type: 'text/html', href: ::TagManager.instance.url_for(account))
-    append_element(author, 'link', nil, rel: :avatar, type: account.avatar_content_type, 'media:width': 120, 'media:height': 120, href: full_asset_url(account.avatar.url(:original))) if account.avatar?
-    append_element(author, 'link', nil, rel: :header, type: account.header_content_type, 'media:width': 700, 'media:height': 335, href: full_asset_url(account.header.url(:original))) if account.header?
-    account.emojis.each do |emoji|
-      append_element(author, 'link', nil, rel: :emoji, href: full_asset_url(emoji.image.url), name: emoji.shortcode)
-    end
-    append_element(author, 'poco:preferredUsername', account.username)
-    append_element(author, 'poco:displayName', account.display_name) if account.display_name?
-    append_element(author, 'poco:note', account.local? ? account.note : strip_tags(account.note)) if account.note?
-    append_element(author, 'mastodon:scope', account.locked? ? :private : :public)
-
-    author
-  end
-
-  def feed(account, stream_entries)
-    feed = Ox::Element.new('feed')
-
-    add_namespaces(feed)
-
-    append_element(feed, 'id', account_url(account, format: 'atom'))
-    append_element(feed, 'title', account.display_name.presence || account.username)
-    append_element(feed, 'subtitle', account.note)
-    append_element(feed, 'updated', account.updated_at.iso8601)
-    append_element(feed, 'logo', full_asset_url(account.avatar.url(:original)))
-
-    feed << author(account)
-
-    append_element(feed, 'link', nil, rel: :alternate, type: 'text/html', href: ::TagManager.instance.url_for(account))
-    append_element(feed, 'link', nil, rel: :self, type: 'application/atom+xml', href: account_url(account, format: 'atom'))
-    append_element(feed, 'link', nil, rel: :next, type: 'application/atom+xml', href: account_url(account, format: 'atom', max_id: stream_entries.last.id)) if stream_entries.size == 20
-
-    stream_entries.each do |stream_entry|
-      feed << entry(stream_entry)
-    end
-
-    feed
-  end
-
-  def entry(stream_entry, root = false)
-    entry = Ox::Element.new('entry')
-
-    add_namespaces(entry) if root
-
-    append_element(entry, 'id', OStatus::TagManager.instance.uri_for(stream_entry.status))
-    append_element(entry, 'published', stream_entry.created_at.iso8601)
-    append_element(entry, 'updated', stream_entry.updated_at.iso8601)
-    append_element(entry, 'title', stream_entry&.status&.title || "#{stream_entry.account.acct} deleted status")
-
-    entry << author(stream_entry.account) if root
-
-    append_element(entry, 'activity:object-type', OStatus::TagManager::TYPES[stream_entry.object_type])
-    append_element(entry, 'activity:verb', OStatus::TagManager::VERBS[stream_entry.verb])
-
-    entry << object(stream_entry.target) if stream_entry.targeted?
-
-    if stream_entry.status.nil?
-      append_element(entry, 'content', 'Deleted status')
-    elsif stream_entry.status.destroyed?
-      append_element(entry, 'content', 'Deleted status')
-      append_element(entry, 'link', nil, rel: :alternate, type: 'application/activity+json', href: ActivityPub::TagManager.instance.uri_for(stream_entry.status)) if stream_entry.account.local?
-    else
-      serialize_status_attributes(entry, stream_entry.status)
-    end
-
-    append_element(entry, 'link', nil, rel: :alternate, type: 'text/html', href: ::TagManager.instance.url_for(stream_entry.status))
-    append_element(entry, 'link', nil, rel: :self, type: 'application/atom+xml', href: account_stream_entry_url(stream_entry.account, stream_entry, format: 'atom'))
-    append_element(entry, 'thr:in-reply-to', nil, ref: OStatus::TagManager.instance.uri_for(stream_entry.thread), href: ::TagManager.instance.url_for(stream_entry.thread)) if stream_entry.threaded?
-    append_element(entry, 'ostatus:conversation', nil, ref: conversation_uri(stream_entry.status.conversation)) unless stream_entry&.status&.conversation_id.nil?
-
-    entry
-  end
-
-  def object(status)
-    object = Ox::Element.new('activity:object')
-
-    append_element(object, 'id', OStatus::TagManager.instance.uri_for(status))
-    append_element(object, 'published', status.created_at.iso8601)
-    append_element(object, 'updated', status.updated_at.iso8601)
-    append_element(object, 'title', status.title)
-
-    object << author(status.account)
-
-    append_element(object, 'activity:object-type', OStatus::TagManager::TYPES[status.object_type])
-    append_element(object, 'activity:verb', OStatus::TagManager::VERBS[status.verb])
-
-    serialize_status_attributes(object, status)
-
-    append_element(object, 'link', nil, rel: :alternate, type: 'text/html', href: ::TagManager.instance.url_for(status))
-    append_element(object, 'thr:in-reply-to', nil, ref: OStatus::TagManager.instance.uri_for(status.thread), href: ::TagManager.instance.url_for(status.thread)) unless status.thread.nil?
-    append_element(object, 'ostatus:conversation', nil, ref: conversation_uri(status.conversation)) unless status.conversation_id.nil?
-
-    object
-  end
-
-  def follow_salmon(follow)
-    entry = Ox::Element.new('entry')
-    add_namespaces(entry)
-
-    description = "#{follow.account.acct} started following #{follow.target_account.acct}"
-
-    append_element(entry, 'id', OStatus::TagManager.instance.unique_tag(follow.created_at, follow.id, 'Follow'))
-    append_element(entry, 'title', description)
-    append_element(entry, 'content', description, type: :html)
-
-    entry << author(follow.account)
-
-    append_element(entry, 'activity:object-type', OStatus::TagManager::TYPES[:activity])
-    append_element(entry, 'activity:verb', OStatus::TagManager::VERBS[:follow])
-
-    object = author(follow.target_account)
-    object.value = 'activity:object'
-
-    entry << object
-    entry
-  end
-
-  def follow_request_salmon(follow_request)
-    entry = Ox::Element.new('entry')
-    add_namespaces(entry)
-
-    append_element(entry, 'id', OStatus::TagManager.instance.unique_tag(follow_request.created_at, follow_request.id, 'FollowRequest'))
-    append_element(entry, 'title', "#{follow_request.account.acct} requested to follow #{follow_request.target_account.acct}")
-
-    entry << author(follow_request.account)
-
-    append_element(entry, 'activity:object-type', OStatus::TagManager::TYPES[:activity])
-    append_element(entry, 'activity:verb', OStatus::TagManager::VERBS[:request_friend])
-
-    object = author(follow_request.target_account)
-    object.value = 'activity:object'
-
-    entry << object
-    entry
-  end
-
-  def authorize_follow_request_salmon(follow_request)
-    entry = Ox::Element.new('entry')
-    add_namespaces(entry)
-
-    append_element(entry, 'id', OStatus::TagManager.instance.unique_tag(Time.now.utc, follow_request.id, 'FollowRequest'))
-    append_element(entry, 'title', "#{follow_request.target_account.acct} authorizes follow request by #{follow_request.account.acct}")
-
-    entry << author(follow_request.target_account)
-
-    append_element(entry, 'activity:object-type', OStatus::TagManager::TYPES[:activity])
-    append_element(entry, 'activity:verb', OStatus::TagManager::VERBS[:authorize])
-
-    object = Ox::Element.new('activity:object')
-    object << author(follow_request.account)
-
-    append_element(object, 'activity:object-type', OStatus::TagManager::TYPES[:activity])
-    append_element(object, 'activity:verb', OStatus::TagManager::VERBS[:request_friend])
-
-    inner_object = author(follow_request.target_account)
-    inner_object.value = 'activity:object'
-
-    object << inner_object
-    entry  << object
-    entry
-  end
-
-  def reject_follow_request_salmon(follow_request)
-    entry = Ox::Element.new('entry')
-    add_namespaces(entry)
-
-    append_element(entry, 'id', OStatus::TagManager.instance.unique_tag(Time.now.utc, follow_request.id, 'FollowRequest'))
-    append_element(entry, 'title', "#{follow_request.target_account.acct} rejects follow request by #{follow_request.account.acct}")
-
-    entry << author(follow_request.target_account)
-
-    append_element(entry, 'activity:object-type', OStatus::TagManager::TYPES[:activity])
-    append_element(entry, 'activity:verb', OStatus::TagManager::VERBS[:reject])
-
-    object = Ox::Element.new('activity:object')
-    object << author(follow_request.account)
-
-    append_element(object, 'activity:object-type', OStatus::TagManager::TYPES[:activity])
-    append_element(object, 'activity:verb', OStatus::TagManager::VERBS[:request_friend])
-
-    inner_object = author(follow_request.target_account)
-    inner_object.value = 'activity:object'
-
-    object << inner_object
-    entry  << object
-    entry
-  end
-
-  def unfollow_salmon(follow)
-    entry = Ox::Element.new('entry')
-    add_namespaces(entry)
-
-    description = "#{follow.account.acct} is no longer following #{follow.target_account.acct}"
-
-    append_element(entry, 'id', OStatus::TagManager.instance.unique_tag(Time.now.utc, follow.id, 'Follow'))
-    append_element(entry, 'title', description)
-    append_element(entry, 'content', description, type: :html)
-
-    entry << author(follow.account)
-
-    append_element(entry, 'activity:object-type', OStatus::TagManager::TYPES[:activity])
-    append_element(entry, 'activity:verb', OStatus::TagManager::VERBS[:unfollow])
-
-    object = author(follow.target_account)
-    object.value = 'activity:object'
-
-    entry << object
-    entry
-  end
-
-  def block_salmon(block)
-    entry = Ox::Element.new('entry')
-    add_namespaces(entry)
-
-    description = "#{block.account.acct} no longer wishes to interact with #{block.target_account.acct}"
-
-    append_element(entry, 'id', OStatus::TagManager.instance.unique_tag(Time.now.utc, block.id, 'Block'))
-    append_element(entry, 'title', description)
-
-    entry << author(block.account)
-
-    append_element(entry, 'activity:object-type', OStatus::TagManager::TYPES[:activity])
-    append_element(entry, 'activity:verb', OStatus::TagManager::VERBS[:block])
-
-    object = author(block.target_account)
-    object.value = 'activity:object'
-
-    entry << object
-    entry
-  end
-
-  def unblock_salmon(block)
-    entry = Ox::Element.new('entry')
-    add_namespaces(entry)
-
-    description = "#{block.account.acct} no longer blocks #{block.target_account.acct}"
-
-    append_element(entry, 'id', OStatus::TagManager.instance.unique_tag(Time.now.utc, block.id, 'Block'))
-    append_element(entry, 'title', description)
-
-    entry << author(block.account)
-
-    append_element(entry, 'activity:object-type', OStatus::TagManager::TYPES[:activity])
-    append_element(entry, 'activity:verb', OStatus::TagManager::VERBS[:unblock])
-
-    object = author(block.target_account)
-    object.value = 'activity:object'
-
-    entry << object
-    entry
-  end
-
-  def favourite_salmon(favourite)
-    entry = Ox::Element.new('entry')
-    add_namespaces(entry)
-
-    description = "#{favourite.account.acct} favourited a status by #{favourite.status.account.acct}"
-
-    append_element(entry, 'id', OStatus::TagManager.instance.unique_tag(favourite.created_at, favourite.id, 'Favourite'))
-    append_element(entry, 'title', description)
-    append_element(entry, 'content', description, type: :html)
-
-    entry << author(favourite.account)
-
-    append_element(entry, 'activity:object-type', OStatus::TagManager::TYPES[:activity])
-    append_element(entry, 'activity:verb', OStatus::TagManager::VERBS[:favorite])
-
-    entry << object(favourite.status)
-
-    append_element(entry, 'thr:in-reply-to', nil, ref: OStatus::TagManager.instance.uri_for(favourite.status), href: ::TagManager.instance.url_for(favourite.status))
-
-    entry
-  end
-
-  def unfavourite_salmon(favourite)
-    entry = Ox::Element.new('entry')
-    add_namespaces(entry)
-
-    description = "#{favourite.account.acct} no longer favourites a status by #{favourite.status.account.acct}"
-
-    append_element(entry, 'id', OStatus::TagManager.instance.unique_tag(Time.now.utc, favourite.id, 'Favourite'))
-    append_element(entry, 'title', description)
-    append_element(entry, 'content', description, type: :html)
-
-    entry << author(favourite.account)
-
-    append_element(entry, 'activity:object-type', OStatus::TagManager::TYPES[:activity])
-    append_element(entry, 'activity:verb', OStatus::TagManager::VERBS[:unfavorite])
-
-    entry << object(favourite.status)
-
-    append_element(entry, 'thr:in-reply-to', nil, ref: OStatus::TagManager.instance.uri_for(favourite.status), href: ::TagManager.instance.url_for(favourite.status))
-
-    entry
-  end
-
-  private
-
-  def append_element(parent, name, content = nil, **attributes)
-    element = Ox::Element.new(name)
-    attributes.each { |k, v| element[k] = sanitize_str(v) }
-    element << sanitize_str(content) unless content.nil?
-    parent  << element
-  end
-
-  def sanitize_str(raw_str)
-    raw_str.to_s
-  end
-
-  def conversation_uri(conversation)
-    return conversation.uri if conversation.uri?
-    OStatus::TagManager.instance.unique_tag(conversation.created_at, conversation.id, 'Conversation')
-  end
-
-  def add_namespaces(parent)
-    parent['xmlns']          = OStatus::TagManager::XMLNS
-    parent['xmlns:thr']      = OStatus::TagManager::THR_XMLNS
-    parent['xmlns:activity'] = OStatus::TagManager::AS_XMLNS
-    parent['xmlns:poco']     = OStatus::TagManager::POCO_XMLNS
-    parent['xmlns:media']    = OStatus::TagManager::MEDIA_XMLNS
-    parent['xmlns:ostatus']  = OStatus::TagManager::OS_XMLNS
-    parent['xmlns:mastodon'] = OStatus::TagManager::MTDN_XMLNS
-  end
-
-  def serialize_status_attributes(entry, status)
-    append_element(entry, 'link', nil, rel: :alternate, type: 'application/activity+json', href: ActivityPub::TagManager.instance.uri_for(status)) if status.account.local?
-
-    append_element(entry, 'summary', status.spoiler_text, 'xml:lang': status.language) if status.spoiler_text?
-    append_element(entry, 'content', Formatter.instance.format(status, inline_poll_options: true).to_str || '.', type: 'html', 'xml:lang': status.language)
-
-    status.active_mentions.sort_by(&:id).each do |mentioned|
-      append_element(entry, 'link', nil, rel: :mentioned, 'ostatus:object-type': OStatus::TagManager::TYPES[:person], href: OStatus::TagManager.instance.uri_for(mentioned.account))
-    end
-
-    append_element(entry, 'link', nil, rel: :mentioned, 'ostatus:object-type': OStatus::TagManager::TYPES[:collection], href: OStatus::TagManager::COLLECTIONS[:public]) if status.public_visibility?
-
-    status.tags.each do |tag|
-      append_element(entry, 'category', nil, term: tag.name)
-    end
-
-    status.media_attachments.each do |media|
-      append_element(entry, 'link', nil, rel: :enclosure, type: media.file_content_type, length: media.file_file_size, href: full_asset_url(media.file.url(:original, false)))
-    end
-
-    append_element(entry, 'category', nil, term: 'nsfw') if status.sensitive? && status.media_attachments.any?
-    append_element(entry, 'mastodon:scope', status.visibility)
-
-    status.emojis.each do |emoji|
-      append_element(entry, 'link', nil, rel: :emoji, href: full_asset_url(emoji.image.url), name: emoji.shortcode)
-    end
-  end
-end
diff --git a/app/lib/status_finder.rb b/app/lib/status_finder.rb
index 4d1aed297..22ced8bf8 100644
--- a/app/lib/status_finder.rb
+++ b/app/lib/status_finder.rb
@@ -13,8 +13,6 @@ class StatusFinder
     raise ActiveRecord::RecordNotFound unless TagManager.instance.local_url?(url)
 
     case recognized_params[:controller]
-    when 'stream_entries'
-      StreamEntry.find(recognized_params[:id]).status
     when 'statuses'
       Status.find(recognized_params[:id])
     else
diff --git a/app/lib/tag_manager.rb b/app/lib/tag_manager.rb
index fb364cb98..daf4f556b 100644
--- a/app/lib/tag_manager.rb
+++ b/app/lib/tag_manager.rb
@@ -33,15 +33,4 @@ class TagManager
     domain = uri.host + (uri.port ? ":#{uri.port}" : '')
     TagManager.instance.web_domain?(domain)
   end
-
-  def url_for(target)
-    return target.url if target.respond_to?(:local?) && !target.local?
-
-    case target.object_type
-    when :person
-      short_account_url(target)
-    when :note, :comment, :activity
-      short_account_status_url(target.account, target)
-    end
-  end
 end
diff --git a/app/mailers/admin_mailer.rb b/app/mailers/admin_mailer.rb
index db154cad5..9ab3e2bbd 100644
--- a/app/mailers/admin_mailer.rb
+++ b/app/mailers/admin_mailer.rb
@@ -3,7 +3,7 @@
 class AdminMailer < ApplicationMailer
   layout 'plain_mailer'
 
-  helper :stream_entries
+  helper :statuses
 
   def new_report(recipient, report)
     @report   = report
diff --git a/app/mailers/notification_mailer.rb b/app/mailers/notification_mailer.rb
index 66fa337c1..723d901fc 100644
--- a/app/mailers/notification_mailer.rb
+++ b/app/mailers/notification_mailer.rb
@@ -1,7 +1,7 @@
 # frozen_string_literal: true
 
 class NotificationMailer < ApplicationMailer
-  helper :stream_entries
+  helper :statuses
 
   add_template_helper RoutingHelper
 
diff --git a/app/models/concerns/account_associations.rb b/app/models/concerns/account_associations.rb
index 70855e054..0921e3252 100644
--- a/app/models/concerns/account_associations.rb
+++ b/app/models/concerns/account_associations.rb
@@ -11,7 +11,6 @@ module AccountAssociations
     has_many :identity_proofs, class_name: 'AccountIdentityProof', dependent: :destroy, inverse_of: :account
 
     # Timelines
-    has_many :stream_entries, inverse_of: :account, dependent: :destroy
     has_many :statuses, inverse_of: :account, dependent: :destroy
     has_many :favourites, inverse_of: :account, dependent: :destroy
     has_many :mentions, inverse_of: :account, dependent: :destroy
diff --git a/app/models/concerns/streamable.rb b/app/models/concerns/streamable.rb
deleted file mode 100644
index 7c9edb8ef..000000000
--- a/app/models/concerns/streamable.rb
+++ /dev/null
@@ -1,43 +0,0 @@
-# frozen_string_literal: true
-
-module Streamable
-  extend ActiveSupport::Concern
-
-  included do
-    has_one :stream_entry, as: :activity
-
-    after_create do
-      account.stream_entries.create!(activity: self, hidden: hidden?) if needs_stream_entry?
-    end
-  end
-
-  def title
-    super
-  end
-
-  def content
-    title
-  end
-
-  def target
-    super
-  end
-
-  def object_type
-    :activity
-  end
-
-  def thread
-    super
-  end
-
-  def hidden?
-    false
-  end
-
-  private
-
-  def needs_stream_entry?
-    account.local?
-  end
-end
diff --git a/app/models/remote_profile.rb b/app/models/remote_profile.rb
deleted file mode 100644
index 742d2b56f..000000000
--- a/app/models/remote_profile.rb
+++ /dev/null
@@ -1,57 +0,0 @@
-# frozen_string_literal: true
-
-class RemoteProfile
-  include ActiveModel::Model
-
-  attr_reader :document
-
-  def initialize(body)
-    @document = Nokogiri::XML.parse(body, nil, 'utf-8')
-  end
-
-  def root
-    @root ||= document.at_xpath('/atom:feed|/atom:entry', atom: OStatus::TagManager::XMLNS)
-  end
-
-  def author
-    @author ||= root.at_xpath('./atom:author|./dfrn:owner', atom: OStatus::TagManager::XMLNS, dfrn: OStatus::TagManager::DFRN_XMLNS)
-  end
-
-  def hub_link
-    @hub_link ||= link_href_from_xml(root, 'hub')
-  end
-
-  def display_name
-    @display_name ||= author.at_xpath('./poco:displayName', poco: OStatus::TagManager::POCO_XMLNS)&.content
-  end
-
-  def note
-    @note ||= author.at_xpath('./atom:summary|./poco:note', atom: OStatus::TagManager::XMLNS, poco: OStatus::TagManager::POCO_XMLNS)&.content
-  end
-
-  def scope
-    @scope ||= author.at_xpath('./mastodon:scope', mastodon: OStatus::TagManager::MTDN_XMLNS)&.content
-  end
-
-  def avatar
-    @avatar ||= link_href_from_xml(author, 'avatar')
-  end
-
-  def header
-    @header ||= link_href_from_xml(author, 'header')
-  end
-
-  def emojis
-    @emojis ||= author.xpath('./xmlns:link[@rel="emoji"]', xmlns: OStatus::TagManager::XMLNS)
-  end
-
-  def locked?
-    scope == 'private'
-  end
-
-  private
-
-  def link_href_from_xml(xml, type)
-    xml.at_xpath(%(./atom:link[@rel="#{type}"]/@href), atom: OStatus::TagManager::XMLNS)&.content
-  end
-end
diff --git a/app/models/status.rb b/app/models/status.rb
index 2258e2d07..906756e85 100644
--- a/app/models/status.rb
+++ b/app/models/status.rb
@@ -28,7 +28,6 @@ class Status < ApplicationRecord
   before_destroy :unlink_from_conversations
 
   include Paginable
-  include Streamable
   include Cacheable
   include StatusThreadingConcern
 
@@ -61,7 +60,6 @@ class Status < ApplicationRecord
   has_and_belongs_to_many :preview_cards
 
   has_one :notification, as: :activity, dependent: :destroy
-  has_one :stream_entry, as: :activity, inverse_of: :status
   has_one :status_stat, inverse_of: :status
   has_one :poll, inverse_of: :status, dependent: :destroy
 
@@ -106,13 +104,11 @@ class Status < ApplicationRecord
                    :status_stat,
                    :tags,
                    :preview_cards,
-                   :stream_entry,
                    :preloadable_poll,
                    account: :account_stat,
                    active_mentions: { account: :account_stat },
                    reblog: [
                      :application,
-                     :stream_entry,
                      :tags,
                      :preview_cards,
                      :media_attachments,
diff --git a/app/models/stream_entry.rb b/app/models/stream_entry.rb
deleted file mode 100644
index 1a9afc5c7..000000000
--- a/app/models/stream_entry.rb
+++ /dev/null
@@ -1,59 +0,0 @@
-# frozen_string_literal: true
-# == Schema Information
-#
-# Table name: stream_entries
-#
-#  id            :bigint(8)        not null, primary key
-#  activity_id   :bigint(8)
-#  activity_type :string
-#  created_at    :datetime         not null
-#  updated_at    :datetime         not null
-#  hidden        :boolean          default(FALSE), not null
-#  account_id    :bigint(8)
-#
-
-class StreamEntry < ApplicationRecord
-  include Paginable
-
-  belongs_to :account, inverse_of: :stream_entries
-  belongs_to :activity, polymorphic: true
-  belongs_to :status, foreign_type: 'Status', foreign_key: 'activity_id', inverse_of: :stream_entry
-
-  validates :account, :activity, presence: true
-
-  STATUS_INCLUDES = [:account, :stream_entry, :conversation, :media_attachments, :tags, mentions: :account, reblog: [:stream_entry, :account, :conversation, :media_attachments, :tags, mentions: :account], thread: [:stream_entry, :account]].freeze
-
-  default_scope { where(activity_type: 'Status') }
-  scope :recent, -> { reorder(id: :desc) }
-  scope :with_includes, -> { includes(:account, status: STATUS_INCLUDES) }
-
-  delegate :target, :title, :content, :thread,
-           to: :status,
-           allow_nil: true
-
-  def object_type
-    orphaned? || targeted? ? :activity : status.object_type
-  end
-
-  def verb
-    orphaned? ? :delete : status.verb
-  end
-
-  def targeted?
-    [:follow, :request_friend, :authorize, :reject, :unfollow, :block, :unblock, :share, :favorite].include? verb
-  end
-
-  def threaded?
-    (verb == :favorite || object_type == :comment) && !thread.nil?
-  end
-
-  def mentions
-    orphaned? ? [] : status.active_mentions.map(&:account)
-  end
-
-  private
-
-  def orphaned?
-    status.nil?
-  end
-end
diff --git a/app/serializers/rest/account_serializer.rb b/app/serializers/rest/account_serializer.rb
index 12adc971c..272e3eb9c 100644
--- a/app/serializers/rest/account_serializer.rb
+++ b/app/serializers/rest/account_serializer.rb
@@ -29,7 +29,7 @@ class REST::AccountSerializer < ActiveModel::Serializer
   end
 
   def url
-    TagManager.instance.url_for(object)
+    ActivityPub::TagManager.instance.url_for(object)
   end
 
   def avatar
diff --git a/app/serializers/rest/status_serializer.rb b/app/serializers/rest/status_serializer.rb
index c9b76cb16..2dc4a1b61 100644
--- a/app/serializers/rest/status_serializer.rb
+++ b/app/serializers/rest/status_serializer.rb
@@ -58,7 +58,7 @@ class REST::StatusSerializer < ActiveModel::Serializer
   end
 
   def uri
-    OStatus::TagManager.instance.uri_for(object)
+    ActivityPub::TagManager.instance.uri_for(object)
   end
 
   def content
@@ -66,7 +66,7 @@ class REST::StatusSerializer < ActiveModel::Serializer
   end
 
   def url
-    TagManager.instance.url_for(object)
+    ActivityPub::TagManager.instance.url_for(object)
   end
 
   def favourited
@@ -132,7 +132,7 @@ class REST::StatusSerializer < ActiveModel::Serializer
     end
 
     def url
-      TagManager.instance.url_for(object.account)
+      ActivityPub::TagManager.instance.url_for(object.account)
     end
 
     def acct
diff --git a/app/serializers/rss/account_serializer.rb b/app/serializers/rss/account_serializer.rb
index 88eca79ed..278affe13 100644
--- a/app/serializers/rss/account_serializer.rb
+++ b/app/serializers/rss/account_serializer.rb
@@ -2,7 +2,7 @@
 
 class RSS::AccountSerializer
   include ActionView::Helpers::NumberHelper
-  include StreamEntriesHelper
+  include StatusesHelper
   include RoutingHelper
 
   def render(account, statuses)
@@ -10,7 +10,7 @@ class RSS::AccountSerializer
 
     builder.title("#{display_name(account)} (@#{account.local_username_and_domain})")
            .description(account_description(account))
-           .link(TagManager.instance.url_for(account))
+           .link(ActivityPub::TagManager.instance.url_for(account))
            .logo(full_pack_url('media/images/logo.svg'))
            .accent_color('2b90d9')
 
@@ -20,7 +20,7 @@ class RSS::AccountSerializer
     statuses.each do |status|
       builder.item do |item|
         item.title(status.title)
-            .link(TagManager.instance.url_for(status))
+            .link(ActivityPub::TagManager.instance.url_for(status))
             .pub_date(status.created_at)
             .description(status.spoiler_text.presence || Formatter.instance.format(status, inline_poll_options: true).to_str)
 
diff --git a/app/serializers/rss/tag_serializer.rb b/app/serializers/rss/tag_serializer.rb
index 644380149..e8562ee87 100644
--- a/app/serializers/rss/tag_serializer.rb
+++ b/app/serializers/rss/tag_serializer.rb
@@ -3,7 +3,7 @@
 class RSS::TagSerializer
   include ActionView::Helpers::NumberHelper
   include ActionView::Helpers::SanitizeHelper
-  include StreamEntriesHelper
+  include StatusesHelper
   include RoutingHelper
 
   def render(tag, statuses)
@@ -18,7 +18,7 @@ class RSS::TagSerializer
     statuses.each do |status|
       builder.item do |item|
         item.title(status.title)
-            .link(TagManager.instance.url_for(status))
+            .link(ActivityPub::TagManager.instance.url_for(status))
             .pub_date(status.created_at)
             .description(status.spoiler_text.presence || Formatter.instance.format(status).to_str)
 
diff --git a/app/services/batched_remove_status_service.rb b/app/services/batched_remove_status_service.rb
index cb66debc8..27dc460a6 100644
--- a/app/services/batched_remove_status_service.rb
+++ b/app/services/batched_remove_status_service.rb
@@ -12,7 +12,7 @@ class BatchedRemoveStatusService < BaseService
   # @param [Hash] options
   # @option [Boolean] :skip_side_effects
   def call(statuses, **options)
-    statuses = Status.where(id: statuses.map(&:id)).includes(:account, :stream_entry).flat_map { |status| [status] + status.reblogs.includes(:account, :stream_entry).to_a }
+    statuses = Status.where(id: statuses.map(&:id)).includes(:account).flat_map { |status| [status] + status.reblogs.includes(:account).to_a }
 
     @mentions = statuses.each_with_object({}) { |s, h| h[s.id] = s.active_mentions.includes(:account).to_a }
     @tags     = statuses.each_with_object({}) { |s, h| h[s.id] = s.tags.pluck(:name) }
diff --git a/app/services/fetch_link_card_service.rb b/app/services/fetch_link_card_service.rb
index 494aaed75..75fbd0e8c 100644
--- a/app/services/fetch_link_card_service.rb
+++ b/app/services/fetch_link_card_service.rb
@@ -84,7 +84,7 @@ class FetchLinkCardService < BaseService
 
   def mention_link?(a)
     @status.mentions.any? do |mention|
-      a['href'] == TagManager.instance.url_for(mention.account)
+      a['href'] == ActivityPub::TagManager.instance.url_for(mention.account)
     end
   end
 
diff --git a/app/services/process_mentions_service.rb b/app/services/process_mentions_service.rb
index da52bff6a..90dca9740 100644
--- a/app/services/process_mentions_service.rb
+++ b/app/services/process_mentions_service.rb
@@ -40,7 +40,7 @@ class ProcessMentionsService < BaseService
   private
 
   def mention_undeliverable?(mentioned_account)
-    mentioned_account.nil? || (!mentioned_account.local? && mentioned_account.ostatus? && @status.stream_entry.hidden?)
+    mentioned_account.nil? || (!mentioned_account.local? && mentioned_account.ostatus?)
   end
 
   def create_notification(mention)
diff --git a/app/services/remove_status_service.rb b/app/services/remove_status_service.rb
index a8c9100b3..6311971ff 100644
--- a/app/services/remove_status_service.rb
+++ b/app/services/remove_status_service.rb
@@ -5,14 +5,13 @@ class RemoveStatusService < BaseService
   include Payloadable
 
   def call(status, **options)
-    @payload      = Oj.dump(event: :delete, payload: status.id.to_s)
-    @status       = status
-    @account      = status.account
-    @tags         = status.tags.pluck(:name).to_a
-    @mentions     = status.active_mentions.includes(:account).to_a
-    @reblogs      = status.reblogs.includes(:account).to_a
-    @stream_entry = status.stream_entry
-    @options      = options
+    @payload  = Oj.dump(event: :delete, payload: status.id.to_s)
+    @status   = status
+    @account  = status.account
+    @tags     = status.tags.pluck(:name).to_a
+    @mentions = status.active_mentions.includes(:account).to_a
+    @reblogs  = status.reblogs.includes(:account).to_a
+    @options  = options
 
     RedisLock.acquire(lock_options) do |lock|
       if lock.acquired?
diff --git a/app/services/resolve_url_service.rb b/app/services/resolve_url_service.rb
index bbdc0a595..f941b489a 100644
--- a/app/services/resolve_url_service.rb
+++ b/app/services/resolve_url_service.rb
@@ -73,10 +73,7 @@ class ResolveURLService < BaseService
 
     return unless recognized_params[:action] == 'show'
 
-    if recognized_params[:controller] == 'stream_entries'
-      status = StreamEntry.find_by(id: recognized_params[:id])&.status
-      check_local_status(status)
-    elsif recognized_params[:controller] == 'statuses'
+    if recognized_params[:controller] == 'statuses'
       status = Status.find_by(id: recognized_params[:id])
       check_local_status(status)
     elsif recognized_params[:controller] == 'accounts'
diff --git a/app/services/suspend_account_service.rb b/app/services/suspend_account_service.rb
index a5ce3dbd9..0ebe0b562 100644
--- a/app/services/suspend_account_service.rb
+++ b/app/services/suspend_account_service.rb
@@ -24,7 +24,6 @@ class SuspendAccountService < BaseService
     report_notes
     scheduled_statuses
     status_pins
-    stream_entries
     subscriptions
   ).freeze
 
diff --git a/app/views/accounts/_moved.html.haml b/app/views/accounts/_moved.html.haml
index 7a777bfea..02fd7bf42 100644
--- a/app/views/accounts/_moved.html.haml
+++ b/app/views/accounts/_moved.html.haml
@@ -3,10 +3,10 @@
 .moved-account-widget
   .moved-account-widget__message
     = fa_icon 'suitcase'
-    = t('accounts.moved_html', name: content_tag(:bdi, content_tag(:strong, display_name(account, custom_emojify: true), class: :emojify)), new_profile_link: link_to(content_tag(:strong, safe_join(['@', content_tag(:span, moved_to_account.acct)])), TagManager.instance.url_for(moved_to_account), class: 'mention'))
+    = t('accounts.moved_html', name: content_tag(:bdi, content_tag(:strong, display_name(account, custom_emojify: true), class: :emojify)), new_profile_link: link_to(content_tag(:strong, safe_join(['@', content_tag(:span, moved_to_account.acct)])), ActivityPub::TagManager.instance.url_for(moved_to_account), class: 'mention'))
 
   .moved-account-widget__card
-    = link_to TagManager.instance.url_for(moved_to_account), class: 'detailed-status__display-name p-author h-card', target: '_blank', rel: 'me noopener' do
+    = link_to ActivityPub::TagManager.instance.url_for(moved_to_account), class: 'detailed-status__display-name p-author h-card', target: '_blank', rel: 'me noopener' do
       .detailed-status__display-avatar
         .account__avatar-overlay
           .account__avatar-overlay-base{ style: "background-image: url('#{moved_to_account.avatar.url(:original)}')" }
diff --git a/app/views/accounts/show.html.haml b/app/views/accounts/show.html.haml
index de7d2a8ba..0dc984dcc 100644
--- a/app/views/accounts/show.html.haml
+++ b/app/views/accounts/show.html.haml
@@ -39,12 +39,12 @@
       - else
         .activity-stream.activity-stream--under-tabs
           - if params[:page].to_i.zero?
-            = render partial: 'stream_entries/status', collection: @pinned_statuses, as: :status, locals: { pinned: true }
+            = render partial: 'statuses/status', collection: @pinned_statuses, as: :status, locals: { pinned: true }
 
           - if @newer_url
             .entry= link_to_more @newer_url
 
-          = render partial: 'stream_entries/status', collection: @statuses, as: :status
+          = render partial: 'statuses/status', collection: @statuses, as: :status
 
           - if @older_url
             .entry= link_to_more @older_url
diff --git a/app/views/admin/accounts/_account.html.haml b/app/views/admin/accounts/_account.html.haml
index eba3ad804..b057d3e42 100644
--- a/app/views/admin/accounts/_account.html.haml
+++ b/app/views/admin/accounts/_account.html.haml
@@ -19,4 +19,4 @@
       = table_link_to 'times', t('admin.accounts.reject'), reject_admin_account_path(account.id), method: :post, data: { confirm: t('admin.accounts.are_you_sure') } if can?(:reject, account.user)
     - else
       = table_link_to 'circle', t('admin.accounts.web'), web_path("accounts/#{account.id}")
-      = table_link_to 'globe', t('admin.accounts.public'), TagManager.instance.url_for(account)
+      = table_link_to 'globe', t('admin.accounts.public'), ActivityPub::TagManager.instance.url_for(account)
diff --git a/app/views/admin/reports/_status.html.haml b/app/views/admin/reports/_status.html.haml
index b3c145120..9376db7ff 100644
--- a/app/views/admin/reports/_status.html.haml
+++ b/app/views/admin/reports/_status.html.haml
@@ -19,7 +19,7 @@
         = react_component :media_gallery, height: 343, sensitive: !current_account&.user&.show_all_media? && status.sensitive? || current_account&.user&.hide_all_media?, 'autoPlayGif': current_account&.user&.setting_auto_play_gif, media: status.proper.media_attachments.map { |a| ActiveModelSerializers::SerializableResource.new(a, serializer: REST::MediaAttachmentSerializer).as_json }
 
     .detailed-status__meta
-      = link_to TagManager.instance.url_for(status), class: 'detailed-status__datetime', target: stream_link_target, rel: 'noopener' do
+      = link_to ActivityPub::TagManager.instance.url_for(status), class: 'detailed-status__datetime', target: stream_link_target, rel: 'noopener' do
         %time.formatted{ datetime: status.created_at.iso8601, title: l(status.created_at) }= l(status.created_at)
       ·
       - if status.reblog?
diff --git a/app/views/application/_card.html.haml b/app/views/application/_card.html.haml
index e6059b035..00254c40c 100644
--- a/app/views/application/_card.html.haml
+++ b/app/views/application/_card.html.haml
@@ -1,4 +1,4 @@
-- account_url = local_assigns[:admin] ? admin_account_path(account.id) : TagManager.instance.url_for(account)
+- account_url = local_assigns[:admin] ? admin_account_path(account.id) : ActivityPub::TagManager.instance.url_for(account)
 
 .card.h-card
   = link_to account_url, target: '_blank', rel: 'noopener' do
diff --git a/app/views/authorize_interactions/_post_follow_actions.html.haml b/app/views/authorize_interactions/_post_follow_actions.html.haml
index 561c60137..dd71160e2 100644
--- a/app/views/authorize_interactions/_post_follow_actions.html.haml
+++ b/app/views/authorize_interactions/_post_follow_actions.html.haml
@@ -1,4 +1,4 @@
 .post-follow-actions
   %div= link_to t('authorize_follow.post_follow.web'), web_url("accounts/#{@resource.id}"), class: 'button button--block'
-  %div= link_to t('authorize_follow.post_follow.return'), TagManager.instance.url_for(@resource), class: 'button button--block'
+  %div= link_to t('authorize_follow.post_follow.return'), ActivityPub::TagManager.instance.url_for(@resource), class: 'button button--block'
   %div= t('authorize_follow.post_follow.close')
diff --git a/app/views/remote_interaction/new.html.haml b/app/views/remote_interaction/new.html.haml
index c8c08991f..2cc0fcb93 100644
--- a/app/views/remote_interaction/new.html.haml
+++ b/app/views/remote_interaction/new.html.haml
@@ -7,7 +7,7 @@
 
     .public-layout
       .activity-stream.activity-stream--highlighted
-        = render 'stream_entries/status', status: @status
+        = render 'statuses/status', status: @status
 
   = simple_form_for @remote_follow, as: :remote_follow, url: remote_interaction_path(@status) do |f|
     = render 'shared/error_messages', object: @remote_follow
diff --git a/app/views/remote_unfollows/_card.html.haml b/app/views/remote_unfollows/_card.html.haml
index 9abcfd37e..80ad3bae2 100644
--- a/app/views/remote_unfollows/_card.html.haml
+++ b/app/views/remote_unfollows/_card.html.haml
@@ -4,7 +4,7 @@
       = image_tag account.avatar.url(:original), alt: '', width: 48, height: 48, class: 'avatar'
 
     %span.display-name
-      - account_url = local_assigns[:admin] ? admin_account_path(account.id) : TagManager.instance.url_for(account)
+      - account_url = local_assigns[:admin] ? admin_account_path(account.id) : ActivityPub::TagManager.instance.url_for(account)
       = link_to account_url, class: 'detailed-status__display-name p-author h-card', target: '_blank', rel: 'noopener' do
         %strong.emojify= display_name(account, custom_emojify: true)
         %span @#{account.acct}
diff --git a/app/views/remote_unfollows/_post_follow_actions.html.haml b/app/views/remote_unfollows/_post_follow_actions.html.haml
index 2a9c062e9..328f7c833 100644
--- a/app/views/remote_unfollows/_post_follow_actions.html.haml
+++ b/app/views/remote_unfollows/_post_follow_actions.html.haml
@@ -1,4 +1,4 @@
 .post-follow-actions
   %div= link_to t('authorize_follow.post_follow.web'), web_url("accounts/#{@account.id}"), class: 'button button--block'
-  %div= link_to t('authorize_follow.post_follow.return'), TagManager.instance.url_for(@account), class: 'button button--block'
+  %div= link_to t('authorize_follow.post_follow.return'), ActivityPub::TagManager.instance.url_for(@account), class: 'button button--block'
   %div= t('authorize_follow.post_follow.close')
diff --git a/app/views/statuses/_attachment_list.html.haml b/app/views/statuses/_attachment_list.html.haml
new file mode 100644
index 000000000..d9706f47b
--- /dev/null
+++ b/app/views/statuses/_attachment_list.html.haml
@@ -0,0 +1,8 @@
+.attachment-list
+  .attachment-list__icon
+    = fa_icon 'link'
+  %ul.attachment-list__list
+    - attachments.each do |media|
+      %li
+        - url = media.remote_url.presence || media.file.url
+        = link_to File.basename(url), url, title: media.description
diff --git a/app/views/statuses/_detailed_status.html.haml b/app/views/statuses/_detailed_status.html.haml
new file mode 100644
index 000000000..8686c2033
--- /dev/null
+++ b/app/views/statuses/_detailed_status.html.haml
@@ -0,0 +1,79 @@
+.detailed-status.detailed-status--flex
+  .p-author.h-card
+    = link_to ActivityPub::TagManager.instance.url_for(status.account), class: 'detailed-status__display-name u-url', target: stream_link_target, rel: 'noopener' do
+      .detailed-status__display-avatar
+        - if current_account&.user&.setting_auto_play_gif || autoplay
+          = image_tag status.account.avatar_original_url, width: 48, height: 48, alt: '', class: 'account__avatar u-photo'
+        - else
+          = image_tag status.account.avatar_static_url, width: 48, height: 48, alt: '', class: 'account__avatar u-photo'
+      %span.display-name
+        %bdi
+          %strong.display-name__html.p-name.emojify= display_name(status.account, custom_emojify: true, autoplay: autoplay)
+        %span.display-name__account
+          = acct(status.account)
+          = fa_icon('lock') if status.account.locked?
+
+  = account_action_button(status.account)
+
+  .status__content.emojify<
+    - if status.spoiler_text?
+      %p{ :style => ('margin-bottom: 0' unless current_account&.user&.setting_expand_spoilers) }<
+        %span.p-summary> #{Formatter.instance.format_spoiler(status, autoplay: autoplay)}&nbsp;
+        %button.status__content__spoiler-link= t('statuses.show_more')
+    .e-content{ lang: status.language, style: "display: #{!current_account&.user&.setting_expand_spoilers && status.spoiler_text? ? 'none' : 'block'}; direction: #{rtl_status?(status) ? 'rtl' : 'ltr'}" }
+      = Formatter.instance.format(status, custom_emojify: true, autoplay: autoplay)
+      - if status.preloadable_poll
+        = react_component :poll, disabled: true, poll: ActiveModelSerializers::SerializableResource.new(status.preloadable_poll, serializer: REST::PollSerializer, scope: current_user, scope_name: :current_user).as_json do
+          = render partial: 'statuses/poll', locals: { status: status, poll: status.preloadable_poll, autoplay: autoplay }
+
+  - if !status.media_attachments.empty?
+    - if status.media_attachments.first.audio_or_video?
+      - video = status.media_attachments.first
+      = react_component :video, src: video.file.url(:original), preview: video.file.url(:small), blurhash: video.blurhash, sensitive: !current_account&.user&.show_all_media? && status.sensitive? || current_account&.user&.hide_all_media?, width: 670, height: 380, detailed: true, inline: true, alt: video.description do
+        = render partial: 'statuses/attachment_list', locals: { attachments: status.media_attachments }
+    - else
+      = react_component :media_gallery, height: 380, sensitive: !current_account&.user&.show_all_media? && status.sensitive? || current_account&.user&.hide_all_media?, standalone: true, 'autoPlayGif': current_account&.user&.setting_auto_play_gif || autoplay, 'reduceMotion': current_account&.user&.setting_reduce_motion, media: status.media_attachments.map { |a| ActiveModelSerializers::SerializableResource.new(a, serializer: REST::MediaAttachmentSerializer).as_json } do
+        = render partial: 'statuses/attachment_list', locals: { attachments: status.media_attachments }
+  - elsif status.preview_card
+    = react_component :card, 'maxDescription': 160, card: ActiveModelSerializers::SerializableResource.new(status.preview_card, serializer: REST::PreviewCardSerializer).as_json
+
+  .detailed-status__meta
+    %data.dt-published{ value: status.created_at.to_time.iso8601 }
+
+    = link_to ActivityPub::TagManager.instance.url_for(status), class: 'detailed-status__datetime u-url u-uid', target: stream_link_target, rel: 'noopener' do
+      %time.formatted{ datetime: status.created_at.iso8601, title: l(status.created_at) }= l(status.created_at)
+    ·
+    - if status.application && @account.user&.setting_show_application
+      - if status.application.website.blank?
+        %strong.detailed-status__application= status.application.name
+      - else
+        = link_to status.application.name, status.application.website, class: 'detailed-status__application', target: '_blank', rel: 'noopener'
+      ·
+    = link_to remote_interaction_path(status, type: :reply), class: 'modal-button detailed-status__link' do
+      - if status.in_reply_to_id.nil?
+        = fa_icon('reply')
+      - else
+        = fa_icon('reply-all')
+      %span.detailed-status__reblogs>= number_to_human status.replies_count, strip_insignificant_zeros: true
+      = " "
+    ·
+    - if status.direct_visibility?
+      %span.detailed-status__link<
+        = fa_icon('envelope')
+    - elsif status.private_visibility? || status.limited_visibility?
+      %span.detailed-status__link<
+        = fa_icon('lock')
+    - else
+      = link_to remote_interaction_path(status, type: :reblog), class: 'modal-button detailed-status__link' do
+        = fa_icon('retweet')
+        %span.detailed-status__reblogs>= number_to_human status.reblogs_count, strip_insignificant_zeros: true
+        = " "
+    ·
+    = link_to remote_interaction_path(status, type: :favourite), class: 'modal-button detailed-status__link' do
+      = fa_icon('star')
+      %span.detailed-status__favorites>= number_to_human status.favourites_count, strip_insignificant_zeros: true
+      = " "
+
+    - if user_signed_in?
+      ·
+      = link_to t('statuses.open_in_web'), web_url("statuses/#{status.id}"), class: 'detailed-status__application', target: '_blank'
diff --git a/app/views/statuses/_og_description.html.haml b/app/views/statuses/_og_description.html.haml
new file mode 100644
index 000000000..a7b18424d
--- /dev/null
+++ b/app/views/statuses/_og_description.html.haml
@@ -0,0 +1,4 @@
+- description = status_description(activity)
+
+%meta{ name: 'description', content: description }/
+= opengraph 'og:description', description
diff --git a/app/views/statuses/_og_image.html.haml b/app/views/statuses/_og_image.html.haml
new file mode 100644
index 000000000..67f9274b6
--- /dev/null
+++ b/app/views/statuses/_og_image.html.haml
@@ -0,0 +1,38 @@
+- if activity.is_a?(Status) && (activity.non_sensitive_with_media? || (activity.with_media? && Setting.preview_sensitive_media))
+  - player_card = false
+  - activity.media_attachments.each do |media|
+    - if media.image?
+      = opengraph 'og:image', full_asset_url(media.file.url(:original))
+      = opengraph 'og:image:type', media.file_content_type
+      - unless media.file.meta.nil?
+        = opengraph 'og:image:width', media.file.meta.dig('original', 'width')
+        = opengraph 'og:image:height', media.file.meta.dig('original', 'height')
+      - if media.description.present?
+        = opengraph 'og:image:alt', media.description
+    - elsif media.video? || media.gifv?
+      - player_card = true
+      = opengraph 'og:image', full_asset_url(media.file.url(:small))
+      = opengraph 'og:image:type', 'image/png'
+      - unless media.file.meta.nil?
+        = opengraph 'og:image:width', media.file.meta.dig('small', 'width')
+        = opengraph 'og:image:height', media.file.meta.dig('small', 'height')
+      = opengraph 'og:video', full_asset_url(media.file.url(:original))
+      = opengraph 'og:video:secure_url', full_asset_url(media.file.url(:original))
+      = opengraph 'og:video:type', media.file_content_type
+      = opengraph 'twitter:player', medium_player_url(media)
+      = opengraph 'twitter:player:stream', full_asset_url(media.file.url(:original))
+      = opengraph 'twitter:player:stream:content_type', media.file_content_type
+      - unless media.file.meta.nil?
+        = opengraph 'og:video:width', media.file.meta.dig('original', 'width')
+        = opengraph 'og:video:height', media.file.meta.dig('original', 'height')
+        = opengraph 'twitter:player:width', media.file.meta.dig('original', 'width')
+        = opengraph 'twitter:player:height', media.file.meta.dig('original', 'height')
+  - if player_card
+    = opengraph 'twitter:card', 'player'
+  - else
+    = opengraph 'twitter:card', 'summary_large_image'
+- else
+  = opengraph 'og:image', full_asset_url(account.avatar.url(:original))
+  = opengraph 'og:image:width', '120'
+  = opengraph 'og:image:height','120'
+  = opengraph 'twitter:card', 'summary'
diff --git a/app/views/statuses/_poll.html.haml b/app/views/statuses/_poll.html.haml
new file mode 100644
index 000000000..ba34890df
--- /dev/null
+++ b/app/views/statuses/_poll.html.haml
@@ -0,0 +1,27 @@
+- show_results = (user_signed_in? && poll.voted?(current_account)) || poll.expired?
+
+.poll
+  %ul
+    - poll.loaded_options.each do |option|
+      %li
+        - if show_results
+          - percent = poll.votes_count > 0 ? 100 * option.votes_count / poll.votes_count : 0
+          %span.poll__chart{ style: "width: #{percent}%" }
+
+          %label.poll__text><
+            %span.poll__number= percent.round
+            = Formatter.instance.format_poll_option(status, option, autoplay: autoplay)
+        - else
+          %label.poll__text><
+            %span.poll__input{ class: poll.multiple? ? 'checkbox' : nil}><
+            = Formatter.instance.format_poll_option(status, option, autoplay: autoplay)
+  .poll__footer
+    - unless show_results
+      %button.button.button-secondary{ disabled: true }
+        = t('statuses.poll.vote')
+
+    %span= t('statuses.poll.total_votes', count: poll.votes_count)
+
+    - unless poll.expires_at.nil?
+      ·
+      %span= l poll.expires_at
diff --git a/app/views/statuses/_simple_status.html.haml b/app/views/statuses/_simple_status.html.haml
new file mode 100644
index 000000000..11220dfcb
--- /dev/null
+++ b/app/views/statuses/_simple_status.html.haml
@@ -0,0 +1,60 @@
+.status
+  .status__info
+    = link_to ActivityPub::TagManager.instance.url_for(status), class: 'status__relative-time u-url u-uid', target: stream_link_target, rel: 'noopener' do
+      %time.time-ago{ datetime: status.created_at.iso8601, title: l(status.created_at) }= l(status.created_at)
+    %data.dt-published{ value: status.created_at.to_time.iso8601 }
+
+    .p-author.h-card
+      = link_to ActivityPub::TagManager.instance.url_for(status.account), class: 'status__display-name u-url', target: stream_link_target, rel: 'noopener' do
+        .status__avatar
+          %div
+            - if current_account&.user&.setting_auto_play_gif || autoplay
+              = image_tag status.account.avatar_original_url, width: 48, height: 48, alt: '', class: 'u-photo account__avatar'
+            - else
+              = image_tag status.account.avatar_static_url, width: 48, height: 48, alt: '', class: 'u-photo account__avatar'
+        %span.display-name
+          %bdi
+            %strong.display-name__html.p-name.emojify= display_name(status.account, custom_emojify: true, autoplay: autoplay)
+          &nbsp;
+          %span.display-name__account
+            = acct(status.account)
+            = fa_icon('lock') if status.account.locked?
+  .status__content.emojify<
+    - if status.spoiler_text?
+      %p{ :style => ('margin-bottom: 0' unless current_account&.user&.setting_expand_spoilers) }<
+        %span.p-summary> #{Formatter.instance.format_spoiler(status, autoplay: autoplay)}&nbsp;
+        %button.status__content__spoiler-link= t('statuses.show_more')
+    .e-content{ lang: status.language, style: "display: #{!current_account&.user&.setting_expand_spoilers && status.spoiler_text? ? 'none' : 'block'}; direction: #{rtl_status?(status) ? 'rtl' : 'ltr'}" }
+      = Formatter.instance.format(status, custom_emojify: true, autoplay: autoplay)
+      - if status.preloadable_poll
+        = react_component :poll, disabled: true, poll: ActiveModelSerializers::SerializableResource.new(status.preloadable_poll, serializer: REST::PollSerializer, scope: current_user, scope_name: :current_user).as_json do
+          = render partial: 'statuses/poll', locals: { status: status, poll: status.preloadable_poll, autoplay: autoplay }
+
+  - if !status.media_attachments.empty?
+    - if status.media_attachments.first.audio_or_video?
+      - video = status.media_attachments.first
+      = react_component :video, src: video.file.url(:original), preview: video.file.url(:small), blurhash: video.blurhash, sensitive: !current_account&.user&.show_all_media? && status.sensitive? || current_account&.user&.hide_all_media?, width: 610, height: 343, inline: true, alt: video.description do
+        = render partial: 'statuses/attachment_list', locals: { attachments: status.media_attachments }
+    - else
+      = react_component :media_gallery, height: 343, sensitive: !current_account&.user&.show_all_media? && status.sensitive? || current_account&.user&.hide_all_media?, 'autoPlayGif': current_account&.user&.setting_auto_play_gif || autoplay, media: status.media_attachments.map { |a| ActiveModelSerializers::SerializableResource.new(a, serializer: REST::MediaAttachmentSerializer).as_json } do
+        = render partial: 'statuses/attachment_list', locals: { attachments: status.media_attachments }
+  - elsif status.preview_card
+    = react_component :card, 'maxDescription': 160, card: ActiveModelSerializers::SerializableResource.new(status.preview_card, serializer: REST::PreviewCardSerializer).as_json
+
+  .status__action-bar
+    .status__action-bar__counter
+      = link_to remote_interaction_path(status, type: :reply), class: 'status__action-bar-button icon-button modal-button', style: 'font-size: 18px; width: 23.1429px; height: 23.1429px; line-height: 23.15px;' do
+        - if status.in_reply_to_id.nil?
+          = fa_icon 'reply fw'
+        - else
+          = fa_icon 'reply-all fw'
+      .status__action-bar__counter__label= obscured_counter status.replies_count
+    = link_to remote_interaction_path(status, type: :reblog), class: 'status__action-bar-button icon-button modal-button', style: 'font-size: 18px; width: 23.1429px; height: 23.1429px; line-height: 23.15px;' do
+      - if status.public_visibility? || status.unlisted_visibility?
+        = fa_icon 'retweet fw'
+      - elsif status.private_visibility?
+        = fa_icon 'lock fw'
+      - else
+        = fa_icon 'envelope fw'
+    = link_to remote_interaction_path(status, type: :favourite), class: 'status__action-bar-button icon-button modal-button', style: 'font-size: 18px; width: 23.1429px; height: 23.1429px; line-height: 23.15px;' do
+      = fa_icon 'star fw'
diff --git a/app/views/statuses/_status.html.haml b/app/views/statuses/_status.html.haml
new file mode 100644
index 000000000..0e3652503
--- /dev/null
+++ b/app/views/statuses/_status.html.haml
@@ -0,0 +1,62 @@
+:ruby
+  pinned          ||= false
+  include_threads ||= false
+  is_predecessor  ||= false
+  is_successor    ||= false
+  direct_reply_id ||= false
+  parent_id       ||= false
+  autoplay        ||= current_account&.user&.setting_auto_play_gif
+  is_direct_parent  = direct_reply_id == status.id
+  is_direct_child   = parent_id == status.in_reply_to_id
+  centered        ||= include_threads && !is_predecessor && !is_successor
+  h_class           = microformats_h_class(status, is_predecessor, is_successor, include_threads)
+  style_classes     = style_classes(status, is_predecessor, is_successor, include_threads)
+  mf_classes        = microformats_classes(status, is_direct_parent, is_direct_child)
+  entry_classes     = h_class + ' ' + mf_classes + ' ' + style_classes
+
+- if status.reply? && include_threads
+  - if @next_ancestor
+    .entry{ class: entry_classes }
+      = link_to_more ActivityPub::TagManager.instance.url_for(@next_ancestor)
+
+  = render partial: 'statuses/status', collection: @ancestors, as: :status, locals: { is_predecessor: true, direct_reply_id: status.in_reply_to_id }, autoplay: autoplay
+
+.entry{ class: entry_classes }
+
+  - if status.reblog?
+    .status__prepend
+      .status__prepend-icon-wrapper
+        %i.status__prepend-icon.fa.fa-fw.fa-retweet
+      %span
+        = link_to ActivityPub::TagManager.instance.url_for(status.account), class: 'status__display-name muted' do
+          %bdi
+            %strong.emojify= display_name(status.account, custom_emojify: true)
+        = t('stream_entries.reblogged')
+  - elsif pinned
+    .status__prepend
+      .status__prepend-icon-wrapper
+        %i.status__prepend-icon.fa.fa-fw.fa-thumb-tack
+      %span
+        = t('stream_entries.pinned')
+
+  = render (centered ? 'statuses/detailed_status' : 'statuses/simple_status'), status: status.proper, autoplay: autoplay
+
+- if include_threads
+  - if @since_descendant_thread_id
+    .entry{ class: entry_classes }
+      = link_to_more short_account_status_url(status.account.username, status, max_descendant_thread_id: @since_descendant_thread_id + 1)
+  - @descendant_threads.each do |thread|
+    = render partial: 'statuses/status', collection: thread[:statuses], as: :status, locals: { is_successor: true, parent_id: status.id }, autoplay: autoplay
+
+    - if thread[:next_status]
+      .entry{ class: entry_classes }
+        = link_to_more ActivityPub::TagManager.instance.url_for(thread[:next_status])
+  - if @next_descendant_thread
+    .entry{ class: entry_classes }
+      = link_to_more short_account_status_url(status.account.username, status, since_descendant_thread_id: @max_descendant_thread_id - 1)
+
+- if include_threads && !embedded_view? && !user_signed_in?
+  .entry{ class: entry_classes }
+    = link_to new_user_session_path, class: 'load-more load-gap' do
+      = fa_icon 'comments'
+      = t('statuses.sign_in_to_participate')
diff --git a/app/views/statuses/embed.html.haml b/app/views/statuses/embed.html.haml
new file mode 100644
index 000000000..6f2ec646f
--- /dev/null
+++ b/app/views/statuses/embed.html.haml
@@ -0,0 +1,3 @@
+- cache @status do
+  .activity-stream.activity-stream--headless
+    = render 'status', status: @status, centered: true, autoplay: @autoplay
diff --git a/app/views/statuses/show.html.haml b/app/views/statuses/show.html.haml
new file mode 100644
index 000000000..704e37a3d
--- /dev/null
+++ b/app/views/statuses/show.html.haml
@@ -0,0 +1,24 @@
+- content_for :page_title do
+  = t('statuses.title', name: display_name(@account), quote: truncate(@status.spoiler_text.presence || @status.text, length: 50, omission: '…', escape: false))
+
+- content_for :header_tags do
+  - if @account.user&.setting_noindex
+    %meta{ name: 'robots', content: 'noindex' }/
+
+  %link{ rel: 'alternate', type: 'application/json+oembed', href: api_oembed_url(url: short_account_status_url(@account, @status), format: 'json') }/
+  %link{ rel: 'alternate', type: 'application/activity+json', href: ActivityPub::TagManager.instance.uri_for(@status) }/
+
+  = opengraph 'og:site_name', site_title
+  = opengraph 'og:type', 'article'
+  = opengraph 'og:title', "#{display_name(@account)} (@#{@account.local_username_and_domain})"
+  = opengraph 'og:url', short_account_status_url(@account, @status)
+
+  = render 'og_description', activity: @status
+  = render 'og_image', activity: @status, account: @account
+
+.grid
+  .column-0
+    .activity-stream.h-entry
+      = render partial: 'status', locals: { status: @status, include_threads: true }
+  .column-1
+    = render 'application/sidebar'
diff --git a/app/views/stream_entries/_attachment_list.html.haml b/app/views/stream_entries/_attachment_list.html.haml
deleted file mode 100644
index d9706f47b..000000000
--- a/app/views/stream_entries/_attachment_list.html.haml
+++ /dev/null
@@ -1,8 +0,0 @@
-.attachment-list
-  .attachment-list__icon
-    = fa_icon 'link'
-  %ul.attachment-list__list
-    - attachments.each do |media|
-      %li
-        - url = media.remote_url.presence || media.file.url
-        = link_to File.basename(url), url, title: media.description
diff --git a/app/views/stream_entries/_detailed_status.html.haml b/app/views/stream_entries/_detailed_status.html.haml
deleted file mode 100644
index 069d0053f..000000000
--- a/app/views/stream_entries/_detailed_status.html.haml
+++ /dev/null
@@ -1,79 +0,0 @@
-.detailed-status.detailed-status--flex
-  .p-author.h-card
-    = link_to TagManager.instance.url_for(status.account), class: 'detailed-status__display-name u-url', target: stream_link_target, rel: 'noopener' do
-      .detailed-status__display-avatar
-        - if current_account&.user&.setting_auto_play_gif || autoplay
-          = image_tag status.account.avatar_original_url, width: 48, height: 48, alt: '', class: 'account__avatar u-photo'
-        - else
-          = image_tag status.account.avatar_static_url, width: 48, height: 48, alt: '', class: 'account__avatar u-photo'
-      %span.display-name
-        %bdi
-          %strong.display-name__html.p-name.emojify= display_name(status.account, custom_emojify: true, autoplay: autoplay)
-        %span.display-name__account
-          = acct(status.account)
-          = fa_icon('lock') if status.account.locked?
-
-  = account_action_button(status.account)
-
-  .status__content.emojify<
-    - if status.spoiler_text?
-      %p{ :style => ('margin-bottom: 0' unless current_account&.user&.setting_expand_spoilers) }<
-        %span.p-summary> #{Formatter.instance.format_spoiler(status, autoplay: autoplay)}&nbsp;
-        %button.status__content__spoiler-link= t('statuses.show_more')
-    .e-content{ lang: status.language, style: "display: #{!current_account&.user&.setting_expand_spoilers && status.spoiler_text? ? 'none' : 'block'}; direction: #{rtl_status?(status) ? 'rtl' : 'ltr'}" }
-      = Formatter.instance.format(status, custom_emojify: true, autoplay: autoplay)
-      - if status.preloadable_poll
-        = react_component :poll, disabled: true, poll: ActiveModelSerializers::SerializableResource.new(status.preloadable_poll, serializer: REST::PollSerializer, scope: current_user, scope_name: :current_user).as_json do
-          = render partial: 'stream_entries/poll', locals: { status: status, poll: status.preloadable_poll, autoplay: autoplay }
-
-  - if !status.media_attachments.empty?
-    - if status.media_attachments.first.audio_or_video?
-      - video = status.media_attachments.first
-      = react_component :video, src: video.file.url(:original), preview: video.file.url(:small), blurhash: video.blurhash, sensitive: !current_account&.user&.show_all_media? && status.sensitive? || current_account&.user&.hide_all_media?, width: 670, height: 380, detailed: true, inline: true, alt: video.description do
-        = render partial: 'stream_entries/attachment_list', locals: { attachments: status.media_attachments }
-    - else
-      = react_component :media_gallery, height: 380, sensitive: !current_account&.user&.show_all_media? && status.sensitive? || current_account&.user&.hide_all_media?, standalone: true, 'autoPlayGif': current_account&.user&.setting_auto_play_gif || autoplay, 'reduceMotion': current_account&.user&.setting_reduce_motion, media: status.media_attachments.map { |a| ActiveModelSerializers::SerializableResource.new(a, serializer: REST::MediaAttachmentSerializer).as_json } do
-        = render partial: 'stream_entries/attachment_list', locals: { attachments: status.media_attachments }
-  - elsif status.preview_card
-    = react_component :card, 'maxDescription': 160, card: ActiveModelSerializers::SerializableResource.new(status.preview_card, serializer: REST::PreviewCardSerializer).as_json
-
-  .detailed-status__meta
-    %data.dt-published{ value: status.created_at.to_time.iso8601 }
-
-    = link_to TagManager.instance.url_for(status), class: 'detailed-status__datetime u-url u-uid', target: stream_link_target, rel: 'noopener' do
-      %time.formatted{ datetime: status.created_at.iso8601, title: l(status.created_at) }= l(status.created_at)
-    ·
-    - if status.application && @account.user&.setting_show_application
-      - if status.application.website.blank?
-        %strong.detailed-status__application= status.application.name
-      - else
-        = link_to status.application.name, status.application.website, class: 'detailed-status__application', target: '_blank', rel: 'noopener'
-      ·
-    = link_to remote_interaction_path(status, type: :reply), class: 'modal-button detailed-status__link' do
-      - if status.in_reply_to_id.nil?
-        = fa_icon('reply')
-      - else
-        = fa_icon('reply-all')
-      %span.detailed-status__reblogs>= number_to_human status.replies_count, strip_insignificant_zeros: true
-      = " "
-    ·
-    - if status.direct_visibility?
-      %span.detailed-status__link<
-        = fa_icon('envelope')
-    - elsif status.private_visibility? || status.limited_visibility?
-      %span.detailed-status__link<
-        = fa_icon('lock')
-    - else
-      = link_to remote_interaction_path(status, type: :reblog), class: 'modal-button detailed-status__link' do
-        = fa_icon('retweet')
-        %span.detailed-status__reblogs>= number_to_human status.reblogs_count, strip_insignificant_zeros: true
-        = " "
-    ·
-    = link_to remote_interaction_path(status, type: :favourite), class: 'modal-button detailed-status__link' do
-      = fa_icon('star')
-      %span.detailed-status__favorites>= number_to_human status.favourites_count, strip_insignificant_zeros: true
-      = " "
-
-    - if user_signed_in?
-      ·
-      = link_to t('statuses.open_in_web'), web_url("statuses/#{status.id}"), class: 'detailed-status__application', target: '_blank'
diff --git a/app/views/stream_entries/_og_description.html.haml b/app/views/stream_entries/_og_description.html.haml
deleted file mode 100644
index a7b18424d..000000000
--- a/app/views/stream_entries/_og_description.html.haml
+++ /dev/null
@@ -1,4 +0,0 @@
-- description = status_description(activity)
-
-%meta{ name: 'description', content: description }/
-= opengraph 'og:description', description
diff --git a/app/views/stream_entries/_og_image.html.haml b/app/views/stream_entries/_og_image.html.haml
deleted file mode 100644
index 67f9274b6..000000000
--- a/app/views/stream_entries/_og_image.html.haml
+++ /dev/null
@@ -1,38 +0,0 @@
-- if activity.is_a?(Status) && (activity.non_sensitive_with_media? || (activity.with_media? && Setting.preview_sensitive_media))
-  - player_card = false
-  - activity.media_attachments.each do |media|
-    - if media.image?
-      = opengraph 'og:image', full_asset_url(media.file.url(:original))
-      = opengraph 'og:image:type', media.file_content_type
-      - unless media.file.meta.nil?
-        = opengraph 'og:image:width', media.file.meta.dig('original', 'width')
-        = opengraph 'og:image:height', media.file.meta.dig('original', 'height')
-      - if media.description.present?
-        = opengraph 'og:image:alt', media.description
-    - elsif media.video? || media.gifv?
-      - player_card = true
-      = opengraph 'og:image', full_asset_url(media.file.url(:small))
-      = opengraph 'og:image:type', 'image/png'
-      - unless media.file.meta.nil?
-        = opengraph 'og:image:width', media.file.meta.dig('small', 'width')
-        = opengraph 'og:image:height', media.file.meta.dig('small', 'height')
-      = opengraph 'og:video', full_asset_url(media.file.url(:original))
-      = opengraph 'og:video:secure_url', full_asset_url(media.file.url(:original))
-      = opengraph 'og:video:type', media.file_content_type
-      = opengraph 'twitter:player', medium_player_url(media)
-      = opengraph 'twitter:player:stream', full_asset_url(media.file.url(:original))
-      = opengraph 'twitter:player:stream:content_type', media.file_content_type
-      - unless media.file.meta.nil?
-        = opengraph 'og:video:width', media.file.meta.dig('original', 'width')
-        = opengraph 'og:video:height', media.file.meta.dig('original', 'height')
-        = opengraph 'twitter:player:width', media.file.meta.dig('original', 'width')
-        = opengraph 'twitter:player:height', media.file.meta.dig('original', 'height')
-  - if player_card
-    = opengraph 'twitter:card', 'player'
-  - else
-    = opengraph 'twitter:card', 'summary_large_image'
-- else
-  = opengraph 'og:image', full_asset_url(account.avatar.url(:original))
-  = opengraph 'og:image:width', '120'
-  = opengraph 'og:image:height','120'
-  = opengraph 'twitter:card', 'summary'
diff --git a/app/views/stream_entries/_poll.html.haml b/app/views/stream_entries/_poll.html.haml
deleted file mode 100644
index ba34890df..000000000
--- a/app/views/stream_entries/_poll.html.haml
+++ /dev/null
@@ -1,27 +0,0 @@
-- show_results = (user_signed_in? && poll.voted?(current_account)) || poll.expired?
-
-.poll
-  %ul
-    - poll.loaded_options.each do |option|
-      %li
-        - if show_results
-          - percent = poll.votes_count > 0 ? 100 * option.votes_count / poll.votes_count : 0
-          %span.poll__chart{ style: "width: #{percent}%" }
-
-          %label.poll__text><
-            %span.poll__number= percent.round
-            = Formatter.instance.format_poll_option(status, option, autoplay: autoplay)
-        - else
-          %label.poll__text><
-            %span.poll__input{ class: poll.multiple? ? 'checkbox' : nil}><
-            = Formatter.instance.format_poll_option(status, option, autoplay: autoplay)
-  .poll__footer
-    - unless show_results
-      %button.button.button-secondary{ disabled: true }
-        = t('statuses.poll.vote')
-
-    %span= t('statuses.poll.total_votes', count: poll.votes_count)
-
-    - unless poll.expires_at.nil?
-      ·
-      %span= l poll.expires_at
diff --git a/app/views/stream_entries/_simple_status.html.haml b/app/views/stream_entries/_simple_status.html.haml
deleted file mode 100644
index 0b924f72f..000000000
--- a/app/views/stream_entries/_simple_status.html.haml
+++ /dev/null
@@ -1,60 +0,0 @@
-.status
-  .status__info
-    = link_to TagManager.instance.url_for(status), class: 'status__relative-time u-url u-uid', target: stream_link_target, rel: 'noopener' do
-      %time.time-ago{ datetime: status.created_at.iso8601, title: l(status.created_at) }= l(status.created_at)
-    %data.dt-published{ value: status.created_at.to_time.iso8601 }
-
-    .p-author.h-card
-      = link_to TagManager.instance.url_for(status.account), class: 'status__display-name u-url', target: stream_link_target, rel: 'noopener' do
-        .status__avatar
-          %div
-            - if current_account&.user&.setting_auto_play_gif || autoplay
-              = image_tag status.account.avatar_original_url, width: 48, height: 48, alt: '', class: 'u-photo account__avatar'
-            - else
-              = image_tag status.account.avatar_static_url, width: 48, height: 48, alt: '', class: 'u-photo account__avatar'
-        %span.display-name
-          %bdi
-            %strong.display-name__html.p-name.emojify= display_name(status.account, custom_emojify: true, autoplay: autoplay)
-          &nbsp;
-          %span.display-name__account
-            = acct(status.account)
-            = fa_icon('lock') if status.account.locked?
-  .status__content.emojify<
-    - if status.spoiler_text?
-      %p{ :style => ('margin-bottom: 0' unless current_account&.user&.setting_expand_spoilers) }<
-        %span.p-summary> #{Formatter.instance.format_spoiler(status, autoplay: autoplay)}&nbsp;
-        %button.status__content__spoiler-link= t('statuses.show_more')
-    .e-content{ lang: status.language, style: "display: #{!current_account&.user&.setting_expand_spoilers && status.spoiler_text? ? 'none' : 'block'}; direction: #{rtl_status?(status) ? 'rtl' : 'ltr'}" }
-      = Formatter.instance.format(status, custom_emojify: true, autoplay: autoplay)
-      - if status.preloadable_poll
-        = react_component :poll, disabled: true, poll: ActiveModelSerializers::SerializableResource.new(status.preloadable_poll, serializer: REST::PollSerializer, scope: current_user, scope_name: :current_user).as_json do
-          = render partial: 'stream_entries/poll', locals: { status: status, poll: status.preloadable_poll, autoplay: autoplay }
-
-  - if !status.media_attachments.empty?
-    - if status.media_attachments.first.audio_or_video?
-      - video = status.media_attachments.first
-      = react_component :video, src: video.file.url(:original), preview: video.file.url(:small), blurhash: video.blurhash, sensitive: !current_account&.user&.show_all_media? && status.sensitive? || current_account&.user&.hide_all_media?, width: 610, height: 343, inline: true, alt: video.description do
-        = render partial: 'stream_entries/attachment_list', locals: { attachments: status.media_attachments }
-    - else
-      = react_component :media_gallery, height: 343, sensitive: !current_account&.user&.show_all_media? && status.sensitive? || current_account&.user&.hide_all_media?, 'autoPlayGif': current_account&.user&.setting_auto_play_gif || autoplay, media: status.media_attachments.map { |a| ActiveModelSerializers::SerializableResource.new(a, serializer: REST::MediaAttachmentSerializer).as_json } do
-        = render partial: 'stream_entries/attachment_list', locals: { attachments: status.media_attachments }
-  - elsif status.preview_card
-    = react_component :card, 'maxDescription': 160, card: ActiveModelSerializers::SerializableResource.new(status.preview_card, serializer: REST::PreviewCardSerializer).as_json
-
-  .status__action-bar
-    .status__action-bar__counter
-      = link_to remote_interaction_path(status, type: :reply), class: 'status__action-bar-button icon-button modal-button', style: 'font-size: 18px; width: 23.1429px; height: 23.1429px; line-height: 23.15px;' do
-        - if status.in_reply_to_id.nil?
-          = fa_icon 'reply fw'
-        - else
-          = fa_icon 'reply-all fw'
-      .status__action-bar__counter__label= obscured_counter status.replies_count
-    = link_to remote_interaction_path(status, type: :reblog), class: 'status__action-bar-button icon-button modal-button', style: 'font-size: 18px; width: 23.1429px; height: 23.1429px; line-height: 23.15px;' do
-      - if status.public_visibility? || status.unlisted_visibility?
-        = fa_icon 'retweet fw'
-      - elsif status.private_visibility?
-        = fa_icon 'lock fw'
-      - else
-        = fa_icon 'envelope fw'
-    = link_to remote_interaction_path(status, type: :favourite), class: 'status__action-bar-button icon-button modal-button', style: 'font-size: 18px; width: 23.1429px; height: 23.1429px; line-height: 23.15px;' do
-      = fa_icon 'star fw'
diff --git a/app/views/stream_entries/_status.html.haml b/app/views/stream_entries/_status.html.haml
deleted file mode 100644
index 83887cd87..000000000
--- a/app/views/stream_entries/_status.html.haml
+++ /dev/null
@@ -1,62 +0,0 @@
-:ruby
-  pinned          ||= false
-  include_threads ||= false
-  is_predecessor  ||= false
-  is_successor    ||= false
-  direct_reply_id ||= false
-  parent_id       ||= false
-  autoplay        ||= current_account&.user&.setting_auto_play_gif
-  is_direct_parent  = direct_reply_id == status.id
-  is_direct_child   = parent_id == status.in_reply_to_id
-  centered        ||= include_threads && !is_predecessor && !is_successor
-  h_class           = microformats_h_class(status, is_predecessor, is_successor, include_threads)
-  style_classes     = style_classes(status, is_predecessor, is_successor, include_threads)
-  mf_classes        = microformats_classes(status, is_direct_parent, is_direct_child)
-  entry_classes     = h_class + ' ' + mf_classes + ' ' + style_classes
-
-- if status.reply? && include_threads
-  - if @next_ancestor
-    .entry{ class: entry_classes }
-      = link_to_more TagManager.instance.url_for(@next_ancestor)
-
-  = render partial: 'stream_entries/status', collection: @ancestors, as: :status, locals: { is_predecessor: true, direct_reply_id: status.in_reply_to_id }, autoplay: autoplay
-
-.entry{ class: entry_classes }
-
-  - if status.reblog?
-    .status__prepend
-      .status__prepend-icon-wrapper
-        %i.status__prepend-icon.fa.fa-fw.fa-retweet
-      %span
-        = link_to TagManager.instance.url_for(status.account), class: 'status__display-name muted' do
-          %bdi
-            %strong.emojify= display_name(status.account, custom_emojify: true)
-        = t('stream_entries.reblogged')
-  - elsif pinned
-    .status__prepend
-      .status__prepend-icon-wrapper
-        %i.status__prepend-icon.fa.fa-fw.fa-thumb-tack
-      %span
-        = t('stream_entries.pinned')
-
-  = render (centered ? 'stream_entries/detailed_status' : 'stream_entries/simple_status'), status: status.proper, autoplay: autoplay
-
-- if include_threads
-  - if @since_descendant_thread_id
-    .entry{ class: entry_classes }
-      = link_to_more short_account_status_url(status.account.username, status, max_descendant_thread_id: @since_descendant_thread_id + 1)
-  - @descendant_threads.each do |thread|
-    = render partial: 'stream_entries/status', collection: thread[:statuses], as: :status, locals: { is_successor: true, parent_id: status.id }, autoplay: autoplay
-
-    - if thread[:next_status]
-      .entry{ class: entry_classes }
-        = link_to_more TagManager.instance.url_for(thread[:next_status])
-  - if @next_descendant_thread
-    .entry{ class: entry_classes }
-      = link_to_more short_account_status_url(status.account.username, status, since_descendant_thread_id: @max_descendant_thread_id - 1)
-
-- if include_threads && !embedded_view? && !user_signed_in?
-  .entry{ class: entry_classes }
-    = link_to new_user_session_path, class: 'load-more load-gap' do
-      = fa_icon 'comments'
-      = t('statuses.sign_in_to_participate')
diff --git a/app/views/stream_entries/embed.html.haml b/app/views/stream_entries/embed.html.haml
deleted file mode 100644
index 4871c101e..000000000
--- a/app/views/stream_entries/embed.html.haml
+++ /dev/null
@@ -1,3 +0,0 @@
-- cache @stream_entry.activity do
-  .activity-stream.activity-stream--headless
-    = render "stream_entries/#{@type}", @type.to_sym => @stream_entry.activity, centered: true, autoplay: @autoplay
diff --git a/app/views/stream_entries/show.html.haml b/app/views/stream_entries/show.html.haml
deleted file mode 100644
index 0e81c4f68..000000000
--- a/app/views/stream_entries/show.html.haml
+++ /dev/null
@@ -1,25 +0,0 @@
-- content_for :page_title do
-  = t('statuses.title', name: display_name(@account), quote: truncate(@stream_entry.activity.spoiler_text.presence || @stream_entry.activity.text, length: 50, omission: '…', escape: false))
-
-- content_for :header_tags do
-  - if @account.user&.setting_noindex
-    %meta{ name: 'robots', content: 'noindex' }/
-
-  %link{ rel: 'alternate', type: 'application/atom+xml', href: account_stream_entry_url(@account, @stream_entry, format: 'atom') }/
-  %link{ rel: 'alternate', type: 'application/json+oembed', href: api_oembed_url(url: account_stream_entry_url(@account, @stream_entry), format: 'json') }/
-  %link{ rel: 'alternate', type: 'application/activity+json', href: ActivityPub::TagManager.instance.uri_for(@stream_entry.activity) }/
-
-  = opengraph 'og:site_name', site_title
-  = opengraph 'og:type', 'article'
-  = opengraph 'og:title', "#{display_name(@account)} (@#{@account.local_username_and_domain})"
-  = opengraph 'og:url', short_account_status_url(@account, @stream_entry.activity)
-
-  = render 'stream_entries/og_description', activity: @stream_entry.activity
-  = render 'stream_entries/og_image', activity: @stream_entry.activity, account: @account
-
-.grid
-  .column-0
-    .activity-stream.h-entry
-      = render partial: "stream_entries/#{@type}", locals: { @type.to_sym => @stream_entry.activity, include_threads: true }
-  .column-1
-    = render 'application/sidebar'
diff --git a/config/routes.rb b/config/routes.rb
index 4b6d464c6..69b495a96 100644
--- a/config/routes.rb
+++ b/config/routes.rb
@@ -45,12 +45,6 @@ Rails.application.routes.draw do
   get '/authorize_follow', to: redirect { |_, request| "/authorize_interaction?#{request.params.to_query}" }
 
   resources :accounts, path: 'users', only: [:show], param: :username do
-    resources :stream_entries, path: 'updates', only: [:show] do
-      member do
-        get :embed
-      end
-    end
-
     get :remote_follow,  to: 'remote_follow#new'
     post :remote_follow, to: 'remote_follow#create'
 
diff --git a/db/post_migrate/20190706233204_drop_stream_entries.rb b/db/post_migrate/20190706233204_drop_stream_entries.rb
new file mode 100644
index 000000000..1fecece05
--- /dev/null
+++ b/db/post_migrate/20190706233204_drop_stream_entries.rb
@@ -0,0 +1,13 @@
+# frozen_string_literal: true
+
+class DropStreamEntries < ActiveRecord::Migration[5.2]
+  disable_ddl_transaction!
+
+  def up
+    drop_table :stream_entries
+  end
+
+  def down
+    raise ActiveRecord::IrreversibleMigration
+  end
+end
diff --git a/db/schema.rb b/db/schema.rb
index 09e6c9fae..2e38fb1f2 100644
--- a/db/schema.rb
+++ b/db/schema.rb
@@ -10,7 +10,7 @@
 #
 # It's strongly recommended that you check this file into your version control system.
 
-ActiveRecord::Schema.define(version: 2019_06_27_222826) do
+ActiveRecord::Schema.define(version: 2019_07_06_233204) do
 
   # These are extensions that must be enabled in order to support this database
   enable_extension "plpgsql"
@@ -647,17 +647,6 @@ ActiveRecord::Schema.define(version: 2019_06_27_222826) do
     t.index ["tag_id", "status_id"], name: "index_statuses_tags_on_tag_id_and_status_id", unique: true
   end
 
-  create_table "stream_entries", force: :cascade do |t|
-    t.bigint "activity_id"
-    t.string "activity_type"
-    t.datetime "created_at", null: false
-    t.datetime "updated_at", null: false
-    t.boolean "hidden", default: false, null: false
-    t.bigint "account_id"
-    t.index ["account_id", "activity_type", "id"], name: "index_stream_entries_on_account_id_and_activity_type_and_id"
-    t.index ["activity_id", "activity_type"], name: "index_stream_entries_on_activity_id_and_activity_type"
-  end
-
   create_table "subscriptions", force: :cascade do |t|
     t.string "callback_url", default: "", null: false
     t.string "secret"
@@ -831,7 +820,6 @@ ActiveRecord::Schema.define(version: 2019_06_27_222826) do
   add_foreign_key "statuses", "statuses", column: "reblog_of_id", on_delete: :cascade
   add_foreign_key "statuses_tags", "statuses", on_delete: :cascade
   add_foreign_key "statuses_tags", "tags", name: "fk_3081861e21", on_delete: :cascade
-  add_foreign_key "stream_entries", "accounts", name: "fk_5659b17554", on_delete: :cascade
   add_foreign_key "subscriptions", "accounts", name: "fk_9847d1cbb5", on_delete: :cascade
   add_foreign_key "tombstones", "accounts", on_delete: :cascade
   add_foreign_key "user_invite_requests", "users", on_delete: :cascade
diff --git a/spec/controllers/accounts_controller_spec.rb b/spec/controllers/accounts_controller_spec.rb
index b728d719f..3d2a0665d 100644
--- a/spec/controllers/accounts_controller_spec.rb
+++ b/spec/controllers/accounts_controller_spec.rb
@@ -48,37 +48,6 @@ RSpec.describe AccountsController, type: :controller do
       end
     end
 
-    context 'atom' do
-      let(:format) { 'atom' }
-      let(:content_type) { 'application/atom+xml' }
-
-      shared_examples 'responsed streams' do
-        it 'assigns @entries' do
-          entries = assigns(:entries).to_a
-          expect(entries.size).to eq expected_statuses.size
-          entries.each.zip(expected_statuses.each) do |entry, expected_status|
-            expect(entry.status).to eq expected_status
-          end
-        end
-      end
-
-      include_examples 'responses'
-
-      context 'without max_id nor since_id' do
-        let(:expected_statuses) { [status7, status6, status5, status4, status3, status2, status1] }
-
-        include_examples 'responsed streams'
-      end
-
-      context 'with max_id and since_id' do
-        let(:max_id) { status4.stream_entry.id }
-        let(:since_id) { status1.stream_entry.id }
-        let(:expected_statuses) { [status3, status2] }
-
-        include_examples 'responsed streams'
-      end
-    end
-
     context 'activitystreams2' do
       let(:format) { 'json' }
       let(:content_type) { 'application/activity+json' }
diff --git a/spec/controllers/api/oembed_controller_spec.rb b/spec/controllers/api/oembed_controller_spec.rb
index 7fee15a35..b9082bde1 100644
--- a/spec/controllers/api/oembed_controller_spec.rb
+++ b/spec/controllers/api/oembed_controller_spec.rb
@@ -9,7 +9,7 @@ RSpec.describe Api::OEmbedController, type: :controller do
   describe 'GET #show' do
     before do
       request.host = Rails.configuration.x.local_domain
-      get :show, params: { url: account_stream_entry_url(alice, status.stream_entry) }, format: :json
+      get :show, params: { url: short_account_status_url(alice, status) }, format: :json
     end
 
     it 'returns http success' do
diff --git a/spec/controllers/application_controller_spec.rb b/spec/controllers/application_controller_spec.rb
index 33cc71087..27946b60f 100644
--- a/spec/controllers/application_controller_spec.rb
+++ b/spec/controllers/application_controller_spec.rb
@@ -360,9 +360,5 @@ describe ApplicationController, type: :controller do
     context 'Status' do
       include_examples 'cacheable', :status, Status
     end
-
-    context 'StreamEntry' do
-      include_examples 'receives :with_includes', :stream_entry, StreamEntry
-    end
   end
 end
diff --git a/spec/controllers/statuses_controller_spec.rb b/spec/controllers/statuses_controller_spec.rb
index 1bb6636c6..95e5c363c 100644
--- a/spec/controllers/statuses_controller_spec.rb
+++ b/spec/controllers/statuses_controller_spec.rb
@@ -55,18 +55,6 @@ describe StatusesController do
         expect(assigns(:status)).to eq status
       end
 
-      it 'assigns @stream_entry' do
-        status = Fabricate(:status)
-        get :show, params: { account_username: status.account.username, id: status.id }
-        expect(assigns(:stream_entry)).to eq status.stream_entry
-      end
-
-      it 'assigns @type' do
-        status = Fabricate(:status)
-        get :show, params: { account_username: status.account.username, id: status.id }
-        expect(assigns(:type)).to eq 'status'
-      end
-
       it 'assigns @ancestors for ancestors of the status if it is a reply' do
         ancestor = Fabricate(:status)
         status = Fabricate(:status, in_reply_to_id: ancestor.id)
@@ -135,10 +123,10 @@ describe StatusesController do
         expect(response).to have_http_status(200)
       end
 
-      it 'renders stream_entries/show' do
+      it 'renders statuses/show' do
         status = Fabricate(:status)
         get :show, params: { account_username: status.account.username, id: status.id }
-        expect(response).to render_template 'stream_entries/show'
+        expect(response).to render_template 'statuses/show'
       end
     end
   end
diff --git a/spec/controllers/stream_entries_controller_spec.rb b/spec/controllers/stream_entries_controller_spec.rb
deleted file mode 100644
index eb7fdf9d7..000000000
--- a/spec/controllers/stream_entries_controller_spec.rb
+++ /dev/null
@@ -1,95 +0,0 @@
-require 'rails_helper'
-
-RSpec.describe StreamEntriesController, type: :controller do
-  render_views
-
-  shared_examples 'before_action' do |route|
-    context 'when account is not suspended and stream_entry is available' do
-      it 'assigns instance variables' do
-        status = Fabricate(:status)
-
-        get route, params: { account_username: status.account.username, id: status.stream_entry.id }
-
-        expect(assigns(:account)).to eq status.account
-        expect(assigns(:stream_entry)).to eq status.stream_entry
-        expect(assigns(:type)).to eq 'status'
-      end
-
-      it 'sets Link headers' do
-        alice = Fabricate(:account, username: 'alice')
-        status = Fabricate(:status, account: alice)
-
-        get route, params: { account_username: alice.username, id: status.stream_entry.id }
-
-        expect(response.headers['Link'].to_s).to eq "<http://test.host/users/alice/updates/#{status.stream_entry.id}.atom>; rel=\"alternate\"; type=\"application/atom+xml\", <https://cb6e6126.ngrok.io/users/alice/statuses/#{status.id}>; rel=\"alternate\"; type=\"application/activity+json\""
-      end
-    end
-
-    context 'when account is suspended' do
-      it 'returns http status 410' do
-        account = Fabricate(:account, suspended: true)
-        status = Fabricate(:status, account: account)
-
-        get route, params: { account_username: account.username, id: status.stream_entry.id }
-
-        expect(response).to have_http_status(410)
-      end
-    end
-
-    context 'when activity is nil' do
-      it 'raises ActiveRecord::RecordNotFound' do
-        account = Fabricate(:account)
-        stream_entry = Fabricate.build(:stream_entry, account: account, activity: nil, activity_type: 'Status')
-        stream_entry.save!(validate: false)
-
-        get route, params: { account_username: account.username, id: stream_entry.id }
-
-        expect(response).to have_http_status(404)
-      end
-    end
-
-    context 'when it is hidden and it is not permitted' do
-      it 'raises ActiveRecord::RecordNotFound' do
-        status = Fabricate(:status)
-        user = Fabricate(:user)
-        status.account.block!(user.account)
-        status.stream_entry.update!(hidden: true)
-
-        sign_in(user)
-        get route, params: { account_username: status.account.username, id: status.stream_entry.id }
-
-        expect(response).to have_http_status(404)
-      end
-    end
-  end
-
-  describe 'GET #show' do
-    include_examples 'before_action', :show
-
-    it 'redirects to status page' do
-      status = Fabricate(:status)
-
-      get :show, params: { account_username: status.account.username, id: status.stream_entry.id }
-
-      expect(response).to redirect_to(short_account_status_url(status.account, status))
-    end
-
-    it 'returns http success with Atom' do
-      status = Fabricate(:status)
-      get :show, params: { account_username: status.account.username, id: status.stream_entry.id }, format: 'atom'
-      expect(response).to have_http_status(200)
-    end
-  end
-
-  describe 'GET #embed' do
-    include_examples 'before_action', :embed
-
-    it 'redirects to new embed page' do
-      status = Fabricate(:status)
-
-      get :embed, params: { account_username: status.account.username, id: status.stream_entry.id }
-
-      expect(response).to redirect_to(embed_short_account_status_url(status.account, status))
-    end
-  end
-end
diff --git a/spec/fabricators/stream_entry_fabricator.rb b/spec/fabricators/stream_entry_fabricator.rb
deleted file mode 100644
index f33822c7c..000000000
--- a/spec/fabricators/stream_entry_fabricator.rb
+++ /dev/null
@@ -1,5 +0,0 @@
-Fabricator(:stream_entry) do
-  account
-  activity { Fabricate(:status) }
-  hidden { [true, false].sample }
-end
diff --git a/spec/helpers/admin/account_moderation_notes_helper_spec.rb b/spec/helpers/admin/account_moderation_notes_helper_spec.rb
index c07f6c4b8..ddfe8b46f 100644
--- a/spec/helpers/admin/account_moderation_notes_helper_spec.rb
+++ b/spec/helpers/admin/account_moderation_notes_helper_spec.rb
@@ -3,7 +3,7 @@
 require 'rails_helper'
 
 RSpec.describe Admin::AccountModerationNotesHelper, type: :helper do
-  include StreamEntriesHelper
+  include StatusesHelper
 
   describe '#admin_account_link_to' do
     context 'account is nil' do
diff --git a/spec/helpers/statuses_helper_spec.rb b/spec/helpers/statuses_helper_spec.rb
new file mode 100644
index 000000000..510955a2f
--- /dev/null
+++ b/spec/helpers/statuses_helper_spec.rb
@@ -0,0 +1,224 @@
+require 'rails_helper'
+
+RSpec.describe StatusesHelper, type: :helper do
+  describe '#display_name' do
+    it 'uses the display name when it exists' do
+      account = Account.new(display_name: "Display", username: "Username")
+
+      expect(helper.display_name(account)).to eq "Display"
+    end
+
+    it 'uses the username when display name is nil' do
+      account = Account.new(display_name: nil, username: "Username")
+
+      expect(helper.display_name(account)).to eq "Username"
+    end
+  end
+
+  describe '#stream_link_target' do
+    it 'returns nil if it is not an embedded view' do
+      set_not_embedded_view
+
+      expect(helper.stream_link_target).to be_nil
+    end
+
+    it 'returns _blank if it is an embedded view' do
+      set_embedded_view
+
+      expect(helper.stream_link_target).to eq '_blank'
+    end
+  end
+
+  describe '#acct' do
+    it 'is fully qualified for embedded local accounts' do
+      allow(Rails.configuration.x).to receive(:local_domain).and_return('local_domain')
+      set_embedded_view
+      account = Account.new(domain: nil, username: 'user')
+
+      acct = helper.acct(account)
+
+      expect(acct).to eq '@user@local_domain'
+    end
+
+    it 'is fully qualified for embedded foreign accounts' do
+      set_embedded_view
+      account = Account.new(domain: 'foreign_server.com', username: 'user')
+
+      acct = helper.acct(account)
+
+      expect(acct).to eq '@user@foreign_server.com'
+    end
+
+    it 'is fully qualified for non embedded foreign accounts' do
+      set_not_embedded_view
+      account = Account.new(domain: 'foreign_server.com', username: 'user')
+
+      acct = helper.acct(account)
+
+      expect(acct).to eq '@user@foreign_server.com'
+    end
+
+    it 'is fully qualified for non embedded local accounts' do
+      allow(Rails.configuration.x).to receive(:local_domain).and_return('local_domain')
+      set_not_embedded_view
+      account = Account.new(domain: nil, username: 'user')
+
+      acct = helper.acct(account)
+
+      expect(acct).to eq '@user@local_domain'
+    end
+  end
+
+  def set_not_embedded_view
+    params[:controller] = "not_#{StatusesHelper::EMBEDDED_CONTROLLER}"
+    params[:action] = "not_#{StatusesHelper::EMBEDDED_ACTION}"
+  end
+
+  def set_embedded_view
+    params[:controller] = StatusesHelper::EMBEDDED_CONTROLLER
+    params[:action] = StatusesHelper::EMBEDDED_ACTION
+  end
+
+  describe '#style_classes' do
+    it do
+      status = double(reblog?: false)
+      classes = helper.style_classes(status, false, false, false)
+
+      expect(classes).to eq 'entry'
+    end
+
+    it do
+      status = double(reblog?: true)
+      classes = helper.style_classes(status, false, false, false)
+
+      expect(classes).to eq 'entry entry-reblog'
+    end
+
+    it do
+      status = double(reblog?: false)
+      classes = helper.style_classes(status, true, false, false)
+
+      expect(classes).to eq 'entry entry-predecessor'
+    end
+
+    it do
+      status = double(reblog?: false)
+      classes = helper.style_classes(status, false, true, false)
+
+      expect(classes).to eq 'entry entry-successor'
+    end
+
+    it do
+      status = double(reblog?: false)
+      classes = helper.style_classes(status, false, false, true)
+
+      expect(classes).to eq 'entry entry-center'
+    end
+
+    it do
+      status = double(reblog?: true)
+      classes = helper.style_classes(status, true, true, true)
+
+      expect(classes).to eq 'entry entry-predecessor entry-reblog entry-successor entry-center'
+    end
+  end
+
+  describe '#microformats_classes' do
+    it do
+      status = double(reblog?: false)
+      classes = helper.microformats_classes(status, false, false)
+
+      expect(classes).to eq ''
+    end
+
+    it do
+      status = double(reblog?: false)
+      classes = helper.microformats_classes(status, true, false)
+
+      expect(classes).to eq 'p-in-reply-to'
+    end
+
+    it do
+      status = double(reblog?: false)
+      classes = helper.microformats_classes(status, false, true)
+
+      expect(classes).to eq 'p-comment'
+    end
+
+    it do
+      status = double(reblog?: true)
+      classes = helper.microformats_classes(status, true, false)
+
+      expect(classes).to eq 'p-in-reply-to p-repost-of'
+    end
+
+    it do
+      status = double(reblog?: true)
+      classes = helper.microformats_classes(status, true, true)
+
+      expect(classes).to eq 'p-in-reply-to p-repost-of p-comment'
+    end
+  end
+
+  describe '#microformats_h_class' do
+    it do
+      status = double(reblog?: false)
+      css_class = helper.microformats_h_class(status, false, false, false)
+
+      expect(css_class).to eq 'h-entry'
+    end
+
+    it do
+      status = double(reblog?: true)
+      css_class = helper.microformats_h_class(status, false, false, false)
+
+      expect(css_class).to eq 'h-cite'
+    end
+
+    it do
+      status = double(reblog?: false)
+      css_class = helper.microformats_h_class(status, true, false, false)
+
+      expect(css_class).to eq 'h-cite'
+    end
+
+    it do
+      status = double(reblog?: false)
+      css_class = helper.microformats_h_class(status, false, true, false)
+
+      expect(css_class).to eq 'h-cite'
+    end
+
+    it do
+      status = double(reblog?: false)
+      css_class = helper.microformats_h_class(status, false, false, true)
+
+      expect(css_class).to eq ''
+    end
+
+    it do
+      status = double(reblog?: true)
+      css_class = helper.microformats_h_class(status, true, true, true)
+
+      expect(css_class).to eq 'h-cite'
+    end
+  end
+
+  describe '#rtl?' do
+    it 'is false if text is empty' do
+      expect(helper).not_to be_rtl ''
+    end
+
+    it 'is false if there are no right to left characters' do
+      expect(helper).not_to be_rtl 'hello world'
+    end
+
+    it 'is false if right to left characters are fewer than 1/3 of total text' do
+      expect(helper).not_to be_rtl 'hello ݟ world'
+    end
+
+    it 'is true if right to left characters are greater than 1/3 of total text' do
+      expect(helper).to be_rtl 'aaݟaaݟ'
+    end
+  end
+end
diff --git a/spec/helpers/stream_entries_helper_spec.rb b/spec/helpers/stream_entries_helper_spec.rb
deleted file mode 100644
index 845b9974e..000000000
--- a/spec/helpers/stream_entries_helper_spec.rb
+++ /dev/null
@@ -1,224 +0,0 @@
-require 'rails_helper'
-
-RSpec.describe StreamEntriesHelper, type: :helper do
-  describe '#display_name' do
-    it 'uses the display name when it exists' do
-      account = Account.new(display_name: "Display", username: "Username")
-
-      expect(helper.display_name(account)).to eq "Display"
-    end
-
-    it 'uses the username when display name is nil' do
-      account = Account.new(display_name: nil, username: "Username")
-
-      expect(helper.display_name(account)).to eq "Username"
-    end
-  end
-
-  describe '#stream_link_target' do
-    it 'returns nil if it is not an embedded view' do
-      set_not_embedded_view
-
-      expect(helper.stream_link_target).to be_nil
-    end
-
-    it 'returns _blank if it is an embedded view' do
-      set_embedded_view
-
-      expect(helper.stream_link_target).to eq '_blank'
-    end
-  end
-
-  describe '#acct' do
-    it 'is fully qualified for embedded local accounts' do
-      allow(Rails.configuration.x).to receive(:local_domain).and_return('local_domain')
-      set_embedded_view
-      account = Account.new(domain: nil, username: 'user')
-
-      acct = helper.acct(account)
-
-      expect(acct).to eq '@user@local_domain'
-    end
-
-    it 'is fully qualified for embedded foreign accounts' do
-      set_embedded_view
-      account = Account.new(domain: 'foreign_server.com', username: 'user')
-
-      acct = helper.acct(account)
-
-      expect(acct).to eq '@user@foreign_server.com'
-    end
-
-    it 'is fully qualified for non embedded foreign accounts' do
-      set_not_embedded_view
-      account = Account.new(domain: 'foreign_server.com', username: 'user')
-
-      acct = helper.acct(account)
-
-      expect(acct).to eq '@user@foreign_server.com'
-    end
-
-    it 'is fully qualified for non embedded local accounts' do
-      allow(Rails.configuration.x).to receive(:local_domain).and_return('local_domain')
-      set_not_embedded_view
-      account = Account.new(domain: nil, username: 'user')
-
-      acct = helper.acct(account)
-
-      expect(acct).to eq '@user@local_domain'
-    end
-  end
-
-  def set_not_embedded_view
-    params[:controller] = "not_#{StreamEntriesHelper::EMBEDDED_CONTROLLER}"
-    params[:action] = "not_#{StreamEntriesHelper::EMBEDDED_ACTION}"
-  end
-
-  def set_embedded_view
-    params[:controller] = StreamEntriesHelper::EMBEDDED_CONTROLLER
-    params[:action] = StreamEntriesHelper::EMBEDDED_ACTION
-  end
-
-  describe '#style_classes' do
-    it do
-      status = double(reblog?: false)
-      classes = helper.style_classes(status, false, false, false)
-
-      expect(classes).to eq 'entry'
-    end
-
-    it do
-      status = double(reblog?: true)
-      classes = helper.style_classes(status, false, false, false)
-
-      expect(classes).to eq 'entry entry-reblog'
-    end
-
-    it do
-      status = double(reblog?: false)
-      classes = helper.style_classes(status, true, false, false)
-
-      expect(classes).to eq 'entry entry-predecessor'
-    end
-
-    it do
-      status = double(reblog?: false)
-      classes = helper.style_classes(status, false, true, false)
-
-      expect(classes).to eq 'entry entry-successor'
-    end
-
-    it do
-      status = double(reblog?: false)
-      classes = helper.style_classes(status, false, false, true)
-
-      expect(classes).to eq 'entry entry-center'
-    end
-
-    it do
-      status = double(reblog?: true)
-      classes = helper.style_classes(status, true, true, true)
-
-      expect(classes).to eq 'entry entry-predecessor entry-reblog entry-successor entry-center'
-    end
-  end
-
-  describe '#microformats_classes' do
-    it do
-      status = double(reblog?: false)
-      classes = helper.microformats_classes(status, false, false)
-
-      expect(classes).to eq ''
-    end
-
-    it do
-      status = double(reblog?: false)
-      classes = helper.microformats_classes(status, true, false)
-
-      expect(classes).to eq 'p-in-reply-to'
-    end
-
-    it do
-      status = double(reblog?: false)
-      classes = helper.microformats_classes(status, false, true)
-
-      expect(classes).to eq 'p-comment'
-    end
-
-    it do
-      status = double(reblog?: true)
-      classes = helper.microformats_classes(status, true, false)
-
-      expect(classes).to eq 'p-in-reply-to p-repost-of'
-    end
-
-    it do
-      status = double(reblog?: true)
-      classes = helper.microformats_classes(status, true, true)
-
-      expect(classes).to eq 'p-in-reply-to p-repost-of p-comment'
-    end
-  end
-
-  describe '#microformats_h_class' do
-    it do
-      status = double(reblog?: false)
-      css_class = helper.microformats_h_class(status, false, false, false)
-
-      expect(css_class).to eq 'h-entry'
-    end
-
-    it do
-      status = double(reblog?: true)
-      css_class = helper.microformats_h_class(status, false, false, false)
-
-      expect(css_class).to eq 'h-cite'
-    end
-
-    it do
-      status = double(reblog?: false)
-      css_class = helper.microformats_h_class(status, true, false, false)
-
-      expect(css_class).to eq 'h-cite'
-    end
-
-    it do
-      status = double(reblog?: false)
-      css_class = helper.microformats_h_class(status, false, true, false)
-
-      expect(css_class).to eq 'h-cite'
-    end
-
-    it do
-      status = double(reblog?: false)
-      css_class = helper.microformats_h_class(status, false, false, true)
-
-      expect(css_class).to eq ''
-    end
-
-    it do
-      status = double(reblog?: true)
-      css_class = helper.microformats_h_class(status, true, true, true)
-
-      expect(css_class).to eq 'h-cite'
-    end
-  end
-
-  describe '#rtl?' do
-    it 'is false if text is empty' do
-      expect(helper).not_to be_rtl ''
-    end
-
-    it 'is false if there are no right to left characters' do
-      expect(helper).not_to be_rtl 'hello world'
-    end
-
-    it 'is false if right to left characters are fewer than 1/3 of total text' do
-      expect(helper).not_to be_rtl 'hello ݟ world'
-    end
-
-    it 'is true if right to left characters are greater than 1/3 of total text' do
-      expect(helper).to be_rtl 'aaݟaaݟ'
-    end
-  end
-end
diff --git a/spec/lib/activitypub/tag_manager_spec.rb b/spec/lib/activitypub/tag_manager_spec.rb
index 6d246629e..1c5c6f0ed 100644
--- a/spec/lib/activitypub/tag_manager_spec.rb
+++ b/spec/lib/activitypub/tag_manager_spec.rb
@@ -143,12 +143,6 @@ RSpec.describe ActivityPub::TagManager do
       expect(subject.uri_to_resource(OStatus::TagManager.instance.uri_for(status), Status)).to eq status
     end
 
-    it 'returns the local status for OStatus StreamEntry URL' do
-      status = Fabricate(:status)
-      stream_entry_url = account_stream_entry_url(status.account, status.stream_entry)
-      expect(subject.uri_to_resource(stream_entry_url, Status)).to eq status
-    end
-
     it 'returns the remote status by matching URI without fragment part' do
       status = Fabricate(:status, uri: 'https://example.com/123')
       expect(subject.uri_to_resource('https://example.com/123#456', Status)).to eq status
diff --git a/spec/lib/ostatus/atom_serializer_spec.rb b/spec/lib/ostatus/atom_serializer_spec.rb
deleted file mode 100644
index 74ab7576f..000000000
--- a/spec/lib/ostatus/atom_serializer_spec.rb
+++ /dev/null
@@ -1,1415 +0,0 @@
-require 'rails_helper'
-
-RSpec.describe OStatus::AtomSerializer do
-  shared_examples 'follow request salmon' do
-    it 'appends author element with account' do
-      account = Fabricate(:account, domain: nil, username: 'username')
-      follow_request = Fabricate(:follow_request, account: account)
-
-      follow_request_salmon = serialize(follow_request)
-
-      expect(follow_request_salmon.author.id.text).to eq 'https://cb6e6126.ngrok.io/users/username'
-    end
-
-    it 'appends activity:object-type element with activity type' do
-      follow_request = Fabricate(:follow_request)
-
-      follow_request_salmon = serialize(follow_request)
-
-      object_type = follow_request_salmon.nodes.find { |node| node.name == 'activity:object-type' }
-      expect(object_type.text).to eq OStatus::TagManager::TYPES[:activity]
-    end
-
-    it 'appends activity:verb element with request_friend type' do
-      follow_request = Fabricate(:follow_request)
-
-      follow_request_salmon = serialize(follow_request)
-
-      verb = follow_request_salmon.nodes.find { |node| node.name == 'activity:verb' }
-      expect(verb.text).to eq OStatus::TagManager::VERBS[:request_friend]
-    end
-
-    it 'appends activity:object with target account' do
-      target_account = Fabricate(:account, domain: 'domain.test', uri: 'https://domain.test/id')
-      follow_request = Fabricate(:follow_request, target_account: target_account)
-
-      follow_request_salmon = serialize(follow_request)
-
-      object = follow_request_salmon.nodes.find { |node| node.name == 'activity:object' }
-      expect(object.id.text).to eq 'https://domain.test/id'
-    end
-  end
-
-  shared_examples 'namespaces' do
-    it 'adds namespaces' do
-      element = serialize
-
-      expect(element['xmlns']).to eq OStatus::TagManager::XMLNS
-      expect(element['xmlns:thr']).to eq OStatus::TagManager::THR_XMLNS
-      expect(element['xmlns:activity']).to eq OStatus::TagManager::AS_XMLNS
-      expect(element['xmlns:poco']).to eq OStatus::TagManager::POCO_XMLNS
-      expect(element['xmlns:media']).to eq OStatus::TagManager::MEDIA_XMLNS
-      expect(element['xmlns:ostatus']).to eq OStatus::TagManager::OS_XMLNS
-      expect(element['xmlns:mastodon']).to eq OStatus::TagManager::MTDN_XMLNS
-    end
-  end
-
-  shared_examples 'no namespaces' do
-    it 'does not add namespaces' do
-      expect(serialize['xmlns']).to eq nil
-    end
-  end
-
-  shared_examples 'status attributes' do
-    it 'appends summary element with spoiler text if present' do
-      status = Fabricate(:status, language: :ca, spoiler_text: 'spoiler text')
-
-      element = serialize(status)
-
-      summary = element.summary
-      expect(summary['xml:lang']).to eq 'ca'
-      expect(summary.text).to eq 'spoiler text'
-    end
-
-    it 'does not append summary element with spoiler text if not present' do
-      status = Fabricate(:status, spoiler_text: '')
-      element = serialize(status)
-      element.nodes.each { |node| expect(node.name).not_to eq 'summary' }
-    end
-
-    it 'appends content element with formatted status' do
-      status = Fabricate(:status, language: :ca, text: 'text')
-
-      element = serialize(status)
-
-      content = element.content
-      expect(content[:type]).to eq 'html'
-      expect(content['xml:lang']).to eq 'ca'
-      expect(content.text).to eq '<p>text</p>'
-    end
-
-    it 'appends link elements for mentioned accounts' do
-      account = Fabricate(:account, username: 'username')
-      status = Fabricate(:status)
-      Fabricate(:mention, account: account, status: status)
-
-      element = serialize(status)
-
-      mentioned = element.nodes.find do |node|
-        node.name == 'link' &&
-          node[:rel] == 'mentioned' &&
-          node['ostatus:object-type'] == OStatus::TagManager::TYPES[:person]
-      end
-
-      expect(mentioned[:href]).to eq 'https://cb6e6126.ngrok.io/users/username'
-    end
-
-    it 'appends link elements for emojis' do
-      Fabricate(:custom_emoji)
-
-      status  = Fabricate(:status, text: ':coolcat:')
-      element = serialize(status)
-      emoji   = element.nodes.find { |node| node.name == 'link' && node[:rel] == 'emoji' }
-
-      expect(emoji[:name]).to eq 'coolcat'
-      expect(emoji[:href]).to_not be_blank
-    end
-  end
-
-  describe 'render' do
-    it 'returns XML with emojis' do
-      element = Ox::Element.new('tag')
-      element << '💩'
-      xml = OStatus::AtomSerializer.render(element)
-
-      expect(xml).to eq "<?xml version=\"1.0\"?>\n<tag>💩</tag>\n"
-    end
-
-    it 'returns XML, stripping invalid characters like \b and \v' do
-      element = Ox::Element.new('tag')
-      element << "im l33t\b haxo\b\vr"
-      xml = OStatus::AtomSerializer.render(element)
-
-      expect(xml).to eq "<?xml version=\"1.0\"?>\n<tag>im l33t haxor</tag>\n"
-    end
-  end
-
-  describe '#author' do
-    context 'when note is present' do
-      it 'appends poco:note element with note for local account' do
-        account = Fabricate(:account, domain: nil, note: '<p>note</p>')
-
-        author = OStatus::AtomSerializer.new.author(account)
-
-        note = author.nodes.find { |node| node.name == 'poco:note' }
-        expect(note.text).to eq '<p>note</p>'
-      end
-
-      it 'appends poco:note element with tags-stripped note for remote account' do
-        account = Fabricate(:account, domain: 'remote', note: '<p>note</p>')
-
-        author = OStatus::AtomSerializer.new.author(account)
-
-        note = author.nodes.find { |node| node.name == 'poco:note' }
-        expect(note.text).to eq 'note'
-      end
-
-      it 'appends summary element with type attribute and simplified note if present' do
-        account = Fabricate(:account, note: 'note')
-        author = OStatus::AtomSerializer.new.author(account)
-        expect(author.summary.text).to eq '<p>note</p>'
-        expect(author.summary[:type]).to eq 'html'
-      end
-    end
-
-    context 'when note is not present' do
-      it 'does not append poco:note element' do
-        account = Fabricate(:account, note: '')
-        author = OStatus::AtomSerializer.new.author(account)
-        author.nodes.each { |node| expect(node.name).not_to eq 'poco:note' }
-      end
-
-      it 'does not append summary element' do
-        account = Fabricate(:account, note: '')
-        author = OStatus::AtomSerializer.new.author(account)
-        author.nodes.each { |node| expect(node.name).not_to eq 'summary' }
-      end
-    end
-
-    it 'returns author element' do
-      account = Fabricate(:account)
-      author = OStatus::AtomSerializer.new.author(account)
-      expect(author.name).to eq 'author'
-    end
-
-    it 'appends activity:object-type element with person type' do
-      account = Fabricate(:account, domain: nil, username: 'username')
-
-      author = OStatus::AtomSerializer.new.author(account)
-
-      object_type = author.nodes.find { |node| node.name == 'activity:object-type' }
-      expect(object_type.text).to eq OStatus::TagManager::TYPES[:person]
-    end
-
-    it 'appends email element with username and domain for local account' do
-      account = Fabricate(:account, username: 'username')
-      author = OStatus::AtomSerializer.new.author(account)
-      expect(author.email.text).to eq 'username@cb6e6126.ngrok.io'
-    end
-
-    it 'appends email element with username and domain for remote user' do
-      account = Fabricate(:account, domain: 'domain', username: 'username')
-      author = OStatus::AtomSerializer.new.author(account)
-      expect(author.email.text).to eq 'username@domain'
-    end
-
-    it 'appends link element for an alternative' do
-      account = Fabricate(:account, domain: nil, username: 'username')
-
-      author = OStatus::AtomSerializer.new.author(account)
-
-      link = author.nodes.find { |node| node.name == 'link' && node[:rel] == 'alternate' && node[:type] == 'text/html' }
-      expect(link[:type]).to eq 'text/html'
-      expect(link[:rel]).to eq 'alternate'
-      expect(link[:href]).to eq 'https://cb6e6126.ngrok.io/@username'
-    end
-
-    it 'has link element for avatar if present' do
-      account = Fabricate(:account, avatar: attachment_fixture('avatar.gif'))
-
-      author = OStatus::AtomSerializer.new.author(account)
-
-      link = author.nodes.find { |node| node.name == 'link' && node[:rel] == 'avatar' }
-      expect(link[:type]).to eq 'image/gif'
-      expect(link['media:width']).to eq '120'
-      expect(link['media:height']).to eq '120'
-      expect(link[:href]).to match  /^https:\/\/cb6e6126.ngrok.io\/system\/accounts\/avatars\/.+\/original\/avatar.gif/
-    end
-
-    it 'does not have link element for avatar if not present' do
-      account = Fabricate(:account, avatar: nil)
-
-      author = OStatus::AtomSerializer.new.author(account)
-
-      author.nodes.each do |node|
-        expect(node[:rel]).not_to eq 'avatar' if node.name == 'link'
-      end
-    end
-
-    it 'appends link element for header if present' do
-      account = Fabricate(:account, header: attachment_fixture('avatar.gif'))
-
-      author = OStatus::AtomSerializer.new.author(account)
-
-      link = author.nodes.find { |node| node.name == 'link' && node[:rel] == 'header' }
-      expect(link[:type]).to eq 'image/gif'
-      expect(link['media:width']).to eq '700'
-      expect(link['media:height']).to eq '335'
-      expect(link[:href]).to match  /^https:\/\/cb6e6126.ngrok.io\/system\/accounts\/headers\/.+\/original\/avatar.gif/
-    end
-
-    it 'does not append link element for header if not present' do
-      account = Fabricate(:account, header: nil)
-
-      author = OStatus::AtomSerializer.new.author(account)
-
-      author.nodes.each do |node|
-        expect(node[:rel]).not_to eq 'header' if node.name == 'link'
-      end
-    end
-
-    it 'appends poco:displayName element with display name if present' do
-      account = Fabricate(:account, display_name: 'display name')
-
-      author = OStatus::AtomSerializer.new.author(account)
-
-      display_name = author.nodes.find { |node| node.name == 'poco:displayName' }
-      expect(display_name.text).to eq 'display name'
-    end
-
-    it 'does not append poco:displayName element with display name if not present' do
-      account = Fabricate(:account, display_name: '')
-      author = OStatus::AtomSerializer.new.author(account)
-      author.nodes.each { |node| expect(node.name).not_to eq 'poco:displayName' }
-    end
-
-    it "appends mastodon:scope element with 'private' if locked" do
-      account = Fabricate(:account, locked: true)
-
-      author = OStatus::AtomSerializer.new.author(account)
-
-      scope = author.nodes.find { |node| node.name == 'mastodon:scope' }
-      expect(scope.text).to eq 'private'
-    end
-
-    it "appends mastodon:scope element with 'public' if unlocked" do
-      account = Fabricate(:account, locked: false)
-
-      author = OStatus::AtomSerializer.new.author(account)
-
-      scope = author.nodes.find { |node| node.name == 'mastodon:scope' }
-      expect(scope.text).to eq 'public'
-    end
-
-    it 'includes URI' do
-      account = Fabricate(:account, domain: nil, username: 'username')
-
-      author = OStatus::AtomSerializer.new.author(account)
-
-      expect(author.id.text).to eq 'https://cb6e6126.ngrok.io/users/username'
-      expect(author.uri.text).to eq 'https://cb6e6126.ngrok.io/users/username'
-    end
-
-    it 'includes username' do
-      account = Fabricate(:account, username: 'username')
-
-      author = OStatus::AtomSerializer.new.author(account)
-
-      name = author.nodes.find { |node| node.name == 'name' }
-      username = author.nodes.find { |node| node.name == 'poco:preferredUsername' }
-      expect(name.text).to eq 'username'
-      expect(username.text).to eq 'username'
-    end
-  end
-
-  describe '#entry' do
-    shared_examples 'not root' do
-      include_examples 'no namespaces' do
-        def serialize
-          subject
-        end
-      end
-
-      it 'does not append author element' do
-        subject.nodes.each { |node| expect(node.name).not_to eq 'author' }
-      end
-    end
-
-    context 'it is root' do
-      include_examples 'namespaces' do
-        def serialize
-          stream_entry = Fabricate(:stream_entry)
-          OStatus::AtomSerializer.new.entry(stream_entry, true)
-        end
-      end
-
-      it 'appends author element' do
-        account = Fabricate(:account, username: 'username')
-        status = Fabricate(:status, account: account)
-
-        entry = OStatus::AtomSerializer.new.entry(status.stream_entry, true)
-
-        expect(entry.author.id.text).to eq 'https://cb6e6126.ngrok.io/users/username'
-      end
-    end
-
-    context 'if status is present' do
-      include_examples 'status attributes' do
-        def serialize(status)
-          OStatus::AtomSerializer.new.entry(status.stream_entry, true)
-        end
-      end
-
-      it 'appends link element for the public collection if status is publicly visible' do
-        status = Fabricate(:status, visibility: :public)
-
-        entry = OStatus::AtomSerializer.new.entry(status.stream_entry)
-
-        mentioned_person = entry.nodes.find do |node|
-          node.name == 'link' &&
-          node[:rel] == 'mentioned' &&
-          node['ostatus:object-type'] == OStatus::TagManager::TYPES[:collection]
-        end
-        expect(mentioned_person[:href]).to eq OStatus::TagManager::COLLECTIONS[:public]
-      end
-
-      it 'does not append link element for the public collection if status is not publicly visible' do
-        status = Fabricate(:status, visibility: :private)
-
-        entry = OStatus::AtomSerializer.new.entry(status.stream_entry)
-
-        entry.nodes.each do |node|
-          if node.name == 'link' &&
-             node[:rel] == 'mentioned' &&
-             node['ostatus:object-type'] == OStatus::TagManager::TYPES[:collection]
-            expect(mentioned_collection[:href]).not_to eq OStatus::TagManager::COLLECTIONS[:public]
-          end
-        end
-      end
-
-      it 'appends category elements for tags' do
-        tag = Fabricate(:tag, name: 'tag')
-        status = Fabricate(:status, tags: [ tag ])
-
-        entry = OStatus::AtomSerializer.new.entry(status.stream_entry)
-
-        expect(entry.category[:term]).to eq 'tag'
-      end
-
-      it 'appends link elements for media attachments' do
-        file = attachment_fixture('attachment.jpg')
-        media_attachment = Fabricate(:media_attachment, file: file)
-        status = Fabricate(:status, media_attachments: [ media_attachment ])
-
-        entry = OStatus::AtomSerializer.new.entry(status.stream_entry)
-
-        enclosure = entry.nodes.find { |node| node.name == 'link' && node[:rel] == 'enclosure' }
-        expect(enclosure[:type]).to eq 'image/jpeg'
-        expect(enclosure[:href]).to match /^https:\/\/cb6e6126.ngrok.io\/system\/media_attachments\/files\/.+\/original\/attachment.jpg$/
-      end
-
-      it 'appends mastodon:scope element with visibility' do
-        status = Fabricate(:status, visibility: :public)
-
-        entry = OStatus::AtomSerializer.new.entry(status.stream_entry)
-
-        scope = entry.nodes.find { |node| node.name == 'mastodon:scope' }
-        expect(scope.text).to eq 'public'
-      end
-    end
-
-    context 'if status is not present' do
-      it 'appends content element saying status is deleted' do
-        status = Fabricate(:status)
-        status.destroy!
-
-        entry = OStatus::AtomSerializer.new.entry(status.stream_entry)
-
-        expect(entry.content.text).to eq 'Deleted status'
-      end
-
-      it 'appends title element saying the status is deleted' do
-        account = Fabricate(:account, username: 'username')
-        status = Fabricate(:status, account: account)
-        status.destroy!
-
-        entry = OStatus::AtomSerializer.new.entry(status.stream_entry)
-
-        expect(entry.title.text).to eq 'username deleted status'
-      end
-    end
-
-    context 'it is not root' do
-      let(:stream_entry) { Fabricate(:stream_entry) }
-      subject { OStatus::AtomSerializer.new.entry(stream_entry, false) }
-      include_examples 'not root'
-    end
-
-    context 'without root parameter' do
-      let(:stream_entry) { Fabricate(:stream_entry) }
-      subject { OStatus::AtomSerializer.new.entry(stream_entry) }
-      include_examples 'not root'
-    end
-
-    it 'returns entry element' do
-      stream_entry = Fabricate(:stream_entry)
-      entry = OStatus::AtomSerializer.new.entry(stream_entry)
-      expect(entry.name).to eq 'entry'
-    end
-
-    it 'appends id element with unique tag' do
-      status = Fabricate(:status, reblog_of_id: nil, created_at: '2000-01-01T00:00:00Z')
-
-      entry = OStatus::AtomSerializer.new.entry(status.stream_entry)
-
-      expect(entry.id.text).to eq "https://cb6e6126.ngrok.io/users/#{status.account.to_param}/statuses/#{status.id}"
-    end
-
-    it 'appends published element with created date' do
-      stream_entry = Fabricate(:stream_entry, created_at: '2000-01-01T00:00:00Z')
-      entry = OStatus::AtomSerializer.new.entry(stream_entry)
-      expect(entry.published.text).to eq '2000-01-01T00:00:00Z'
-    end
-
-    it 'appends updated element with updated date' do
-      stream_entry = Fabricate(:stream_entry, updated_at: '2000-01-01T00:00:00Z')
-      entry = OStatus::AtomSerializer.new.entry(stream_entry)
-      expect(entry.updated.text).to eq '2000-01-01T00:00:00Z'
-    end
-
-    it 'appends title element with status title' do
-      account = Fabricate(:account, username: 'username')
-      status = Fabricate(:status, account: account, reblog_of_id: nil)
-      entry = OStatus::AtomSerializer.new.entry(status.stream_entry)
-      expect(entry.title.text).to eq 'New status by username'
-    end
-
-    it 'appends activity:object-type element with object type' do
-      status = Fabricate(:status)
-      entry = OStatus::AtomSerializer.new.entry(status.stream_entry)
-      object_type = entry.nodes.find { |node| node.name == 'activity:object-type' }
-      expect(object_type.text).to eq OStatus::TagManager::TYPES[:note]
-    end
-
-    it 'appends activity:verb element with object type' do
-      status = Fabricate(:status)
-
-      entry = OStatus::AtomSerializer.new.entry(status.stream_entry)
-
-      object_type = entry.nodes.find { |node| node.name == 'activity:verb' }
-      expect(object_type.text).to eq OStatus::TagManager::VERBS[:post]
-    end
-
-    it 'appends activity:object element with target if present' do
-      reblogged = Fabricate(:status, created_at: '2000-01-01T00:00:00Z')
-      reblog = Fabricate(:status, reblog: reblogged)
-
-      entry = OStatus::AtomSerializer.new.entry(reblog.stream_entry)
-
-      object = entry.nodes.find { |node| node.name == 'activity:object' }
-      expect(object.id.text).to eq "https://cb6e6126.ngrok.io/users/#{reblogged.account.to_param}/statuses/#{reblogged.id}"
-    end
-
-    it 'does not append activity:object element if target is not present' do
-      status = Fabricate(:status, reblog_of_id: nil)
-      entry = OStatus::AtomSerializer.new.entry(status.stream_entry)
-      entry.nodes.each { |node| expect(node.name).not_to eq 'activity:object' }
-    end
-
-    it 'appends link element for an alternative' do
-      account = Fabricate(:account, username: 'username')
-      status = Fabricate(:status, account: account)
-
-      entry = OStatus::AtomSerializer.new.entry(status.stream_entry)
-
-      link = entry.nodes.find { |node| node.name == 'link' && node[:rel] == 'alternate' && node[:type] == 'text/html' }
-      expect(link[:type]).to eq 'text/html'
-      expect(link[:href]).to eq "https://cb6e6126.ngrok.io/@username/#{status.id}"
-    end
-
-    it 'appends link element for itself' do
-      account = Fabricate(:account, username: 'username')
-      status = Fabricate(:status, account: account)
-
-      entry = OStatus::AtomSerializer.new.entry(status.stream_entry)
-
-      link = entry.nodes.find { |node| node.name == 'link' && node[:rel] == 'self' }
-      expect(link[:type]).to eq 'application/atom+xml'
-      expect(link[:href]).to eq "https://cb6e6126.ngrok.io/users/username/updates/#{status.stream_entry.id}.atom"
-    end
-
-    it 'appends thr:in-reply-to element if threaded' do
-      in_reply_to_status = Fabricate(:status, created_at: '2000-01-01T00:00:00Z', reblog_of_id: nil)
-      reply_status = Fabricate(:status, in_reply_to_id: in_reply_to_status.id)
-
-      entry = OStatus::AtomSerializer.new.entry(reply_status.stream_entry)
-
-      in_reply_to = entry.nodes.find { |node| node.name == 'thr:in-reply-to' }
-      expect(in_reply_to[:ref]).to eq "https://cb6e6126.ngrok.io/users/#{in_reply_to_status.account.to_param}/statuses/#{in_reply_to_status.id}"
-    end
-
-    it 'does not append thr:in-reply-to element if not threaded' do
-      status = Fabricate(:status)
-      entry = OStatus::AtomSerializer.new.entry(status.stream_entry)
-      entry.nodes.each { |node| expect(node.name).not_to eq 'thr:in-reply-to' }
-    end
-
-    it 'appends ostatus:conversation if conversation id is present' do
-      status = Fabricate(:status)
-      status.conversation.update!(created_at: '2000-01-01T00:00:00Z')
-
-      entry = OStatus::AtomSerializer.new.entry(status.stream_entry)
-
-      conversation = entry.nodes.find { |node| node.name == 'ostatus:conversation' }
-      expect(conversation[:ref]).to eq "tag:cb6e6126.ngrok.io,2000-01-01:objectId=#{status.conversation_id}:objectType=Conversation"
-    end
-
-    it 'does not append ostatus:conversation if conversation id is not present' do
-      status = Fabricate.build(:status, conversation_id: nil)
-      status.save!(validate: false)
-
-      entry = OStatus::AtomSerializer.new.entry(status.stream_entry)
-
-      entry.nodes.each { |node| expect(node.name).not_to eq 'ostatus:conversation' }
-    end
-  end
-
-  describe '#feed' do
-    include_examples 'namespaces' do
-      def serialize
-        account = Fabricate(:account)
-        OStatus::AtomSerializer.new.feed(account, [])
-      end
-    end
-
-    it 'returns feed element' do
-      account = Fabricate(:account)
-      feed = OStatus::AtomSerializer.new.feed(account, [])
-      expect(feed.name).to eq 'feed'
-    end
-
-    it 'appends id element with account Atom URL' do
-      account = Fabricate(:account, username: 'username')
-      feed = OStatus::AtomSerializer.new.feed(account, [])
-      expect(feed.id.text).to eq 'https://cb6e6126.ngrok.io/users/username.atom'
-    end
-
-    it 'appends title element with account display name if present' do
-      account = Fabricate(:account, display_name: 'display name')
-      feed = OStatus::AtomSerializer.new.feed(account, [])
-      expect(feed.title.text).to eq 'display name'
-    end
-
-    it 'does not append title element with account username if account display name is not present' do
-      account = Fabricate(:account, display_name: '', username: 'username')
-      feed = OStatus::AtomSerializer.new.feed(account, [])
-      expect(feed.title.text).to eq 'username'
-    end
-
-    it 'appends subtitle element with account note' do
-      account = Fabricate(:account, note: 'note')
-      feed = OStatus::AtomSerializer.new.feed(account, [])
-      expect(feed.subtitle.text).to eq 'note'
-    end
-
-    it 'appends updated element with date account got updated' do
-      account = Fabricate(:account, updated_at: '2000-01-01T00:00:00Z')
-      feed = OStatus::AtomSerializer.new.feed(account, [])
-      expect(feed.updated.text).to eq '2000-01-01T00:00:00Z'
-    end
-
-    it 'appends logo element with full asset URL for original account avatar' do
-      account = Fabricate(:account, avatar: attachment_fixture('avatar.gif'))
-      feed = OStatus::AtomSerializer.new.feed(account, [])
-      expect(feed.logo.text).to match /^https:\/\/cb6e6126.ngrok.io\/system\/accounts\/avatars\/.+\/original\/avatar.gif/
-    end
-
-    it 'appends author element' do
-      account = Fabricate(:account, username: 'username')
-      feed = OStatus::AtomSerializer.new.feed(account, [])
-      expect(feed.author.id.text).to eq 'https://cb6e6126.ngrok.io/users/username'
-    end
-
-    it 'appends link element for an alternative' do
-      account = Fabricate(:account, username: 'username')
-
-      feed = OStatus::AtomSerializer.new.feed(account, [])
-
-      link = feed.nodes.find { |node| node.name == 'link' && node[:rel] == 'alternate' && node[:type] == 'text/html' }
-      expect(link[:type]).to eq 'text/html'
-      expect(link[:href]).to eq 'https://cb6e6126.ngrok.io/@username'
-    end
-
-    it 'appends link element for itself' do
-      account = Fabricate(:account, username: 'username')
-
-      feed = OStatus::AtomSerializer.new.feed(account, [])
-
-      link = feed.nodes.find { |node| node.name == 'link' && node[:rel] == 'self' }
-      expect(link[:type]).to eq 'application/atom+xml'
-      expect(link[:href]).to eq 'https://cb6e6126.ngrok.io/users/username.atom'
-    end
-
-    it 'appends link element for the next if it has 20 stream entries' do
-      account = Fabricate(:account, username: 'username')
-      stream_entry = Fabricate(:stream_entry)
-
-      feed = OStatus::AtomSerializer.new.feed(account, Array.new(20, stream_entry))
-
-      link = feed.nodes.find { |node| node.name == 'link' && node[:rel] == 'next' }
-      expect(link[:type]).to eq 'application/atom+xml'
-      expect(link[:href]).to eq "https://cb6e6126.ngrok.io/users/username.atom?max_id=#{stream_entry.id}"
-    end
-
-    it 'does not append link element for the next if it does not have 20 stream entries' do
-      account = Fabricate(:account, username: 'username')
-
-      feed = OStatus::AtomSerializer.new.feed(account, [])
-
-      feed.nodes.each do |node|
-        expect(node[:rel]).not_to eq 'next' if node.name == 'link'
-      end
-    end
-
-    it 'appends stream entries' do
-      account = Fabricate(:account, username: 'username')
-      status = Fabricate(:status, account: account)
-
-      feed = OStatus::AtomSerializer.new.feed(account, [status.stream_entry])
-
-      expect(feed.entry.title.text).to eq 'New status by username'
-    end
-  end
-
-  describe '#block_salmon' do
-    include_examples 'namespaces' do
-      def serialize
-        block = Fabricate(:block)
-        OStatus::AtomSerializer.new.block_salmon(block)
-      end
-    end
-
-    it 'returns entry element' do
-      block = Fabricate(:block)
-      block_salmon = OStatus::AtomSerializer.new.block_salmon(block)
-      expect(block_salmon.name).to eq 'entry'
-    end
-
-    it 'appends id element with unique tag' do
-      block = Fabricate(:block)
-
-      time_before = Time.zone.now
-      block_salmon = OStatus::AtomSerializer.new.block_salmon(block)
-      time_after = Time.zone.now
-
-      expect(block_salmon.id.text).to(
-        eq(OStatus::TagManager.instance.unique_tag(time_before.utc, block.id, 'Block'))
-          .or(eq(OStatus::TagManager.instance.unique_tag(time_after.utc, block.id, 'Block')))
-      )
-    end
-
-    it 'appends title element with description' do
-      account = Fabricate(:account, domain: nil, username: 'account')
-      target_account = Fabricate(:account, domain: 'remote', username: 'target_account')
-      block = Fabricate(:block, account: account, target_account: target_account)
-
-      block_salmon = OStatus::AtomSerializer.new.block_salmon(block)
-
-      expect(block_salmon.title.text).to eq 'account no longer wishes to interact with target_account@remote'
-    end
-
-    it 'appends author element with account' do
-      account = Fabricate(:account, domain: nil, username: 'account')
-      block = Fabricate(:block, account: account)
-
-      block_salmon = OStatus::AtomSerializer.new.block_salmon(block)
-
-      expect(block_salmon.author.id.text).to eq 'https://cb6e6126.ngrok.io/users/account'
-    end
-
-    it 'appends activity:object-type element with activity type' do
-      block = Fabricate(:block)
-
-      block_salmon = OStatus::AtomSerializer.new.block_salmon(block)
-
-      object_type = block_salmon.nodes.find { |node| node.name == 'activity:object-type' }
-      expect(object_type.text).to eq OStatus::TagManager::TYPES[:activity]
-    end
-
-    it 'appends activity:verb element with block' do
-      block = Fabricate(:block)
-
-      block_salmon = OStatus::AtomSerializer.new.block_salmon(block)
-
-      verb = block_salmon.nodes.find { |node| node.name == 'activity:verb' }
-      expect(verb.text).to eq OStatus::TagManager::VERBS[:block]
-    end
-
-    it 'appends activity:object element with target account' do
-      target_account = Fabricate(:account, domain: 'domain.test', uri: 'https://domain.test/id')
-      block = Fabricate(:block, target_account: target_account)
-
-      block_salmon = OStatus::AtomSerializer.new.block_salmon(block)
-
-      object = block_salmon.nodes.find { |node| node.name == 'activity:object' }
-      expect(object.id.text).to eq 'https://domain.test/id'
-    end
-  end
-
-  describe '#unblock_salmon' do
-    include_examples 'namespaces' do
-      def serialize
-        block = Fabricate(:block)
-        OStatus::AtomSerializer.new.unblock_salmon(block)
-      end
-    end
-
-    it 'returns entry element' do
-      block = Fabricate(:block)
-      unblock_salmon = OStatus::AtomSerializer.new.unblock_salmon(block)
-      expect(unblock_salmon.name).to eq 'entry'
-    end
-
-    it 'appends id element with unique tag' do
-      block = Fabricate(:block)
-
-      time_before = Time.zone.now
-      unblock_salmon = OStatus::AtomSerializer.new.unblock_salmon(block)
-      time_after = Time.zone.now
-
-      expect(unblock_salmon.id.text).to(
-        eq(OStatus::TagManager.instance.unique_tag(time_before.utc, block.id, 'Block'))
-          .or(eq(OStatus::TagManager.instance.unique_tag(time_after.utc, block.id, 'Block')))
-      )
-    end
-
-    it 'appends title element with description' do
-      account = Fabricate(:account, domain: nil, username: 'account')
-      target_account = Fabricate(:account, domain: 'remote', username: 'target_account')
-      block = Fabricate(:block, account: account, target_account: target_account)
-
-      unblock_salmon = OStatus::AtomSerializer.new.unblock_salmon(block)
-
-      expect(unblock_salmon.title.text).to eq 'account no longer blocks target_account@remote'
-    end
-
-    it 'appends author element with account' do
-      account = Fabricate(:account, domain: nil, username: 'account')
-      block = Fabricate(:block, account: account)
-
-      unblock_salmon = OStatus::AtomSerializer.new.unblock_salmon(block)
-
-      expect(unblock_salmon.author.id.text).to eq 'https://cb6e6126.ngrok.io/users/account'
-    end
-
-    it 'appends activity:object-type element with activity type' do
-      block = Fabricate(:block)
-
-      unblock_salmon = OStatus::AtomSerializer.new.unblock_salmon(block)
-
-      object_type = unblock_salmon.nodes.find { |node| node.name == 'activity:object-type' }
-      expect(object_type.text).to eq OStatus::TagManager::TYPES[:activity]
-    end
-
-    it 'appends activity:verb element with block' do
-      block = Fabricate(:block)
-
-      unblock_salmon = OStatus::AtomSerializer.new.unblock_salmon(block)
-
-      verb = unblock_salmon.nodes.find { |node| node.name == 'activity:verb' }
-      expect(verb.text).to eq OStatus::TagManager::VERBS[:unblock]
-    end
-
-    it 'appends activity:object element with target account' do
-      target_account = Fabricate(:account, domain: 'domain.test', uri: 'https://domain.test/id')
-      block = Fabricate(:block, target_account: target_account)
-
-      unblock_salmon = OStatus::AtomSerializer.new.unblock_salmon(block)
-
-      object = unblock_salmon.nodes.find { |node| node.name == 'activity:object' }
-      expect(object.id.text).to eq 'https://domain.test/id'
-    end
-  end
-
-  describe '#favourite_salmon' do
-    include_examples 'namespaces' do
-      def serialize
-        favourite = Fabricate(:favourite)
-        OStatus::AtomSerializer.new.favourite_salmon(favourite)
-      end
-    end
-
-    it 'returns entry element' do
-      favourite = Fabricate(:favourite)
-      favourite_salmon = OStatus::AtomSerializer.new.favourite_salmon(favourite)
-      expect(favourite_salmon.name).to eq 'entry'
-    end
-
-    it 'appends id element with unique tag' do
-      favourite = Fabricate(:favourite, created_at: '2000-01-01T00:00:00Z')
-      favourite_salmon = OStatus::AtomSerializer.new.favourite_salmon(favourite)
-      expect(favourite_salmon.id.text).to eq "tag:cb6e6126.ngrok.io,2000-01-01:objectId=#{favourite.id}:objectType=Favourite"
-    end
-
-    it 'appends author element with account' do
-      account = Fabricate(:account, domain: nil, username: 'username')
-      favourite = Fabricate(:favourite, account: account)
-
-      favourite_salmon = OStatus::AtomSerializer.new.favourite_salmon(favourite)
-
-      expect(favourite_salmon.author.id.text).to eq 'https://cb6e6126.ngrok.io/users/username'
-    end
-
-    it 'appends activity:object-type element with activity type' do
-      favourite = Fabricate(:favourite)
-
-      favourite_salmon = OStatus::AtomSerializer.new.favourite_salmon(favourite)
-
-      object_type = favourite_salmon.nodes.find { |node| node.name == 'activity:object-type' }
-      expect(object_type.text).to eq 'http://activitystrea.ms/schema/1.0/activity'
-    end
-
-    it 'appends activity:verb element with favorite' do
-      favourite = Fabricate(:favourite)
-
-      favourite_salmon = OStatus::AtomSerializer.new.favourite_salmon(favourite)
-
-      verb = favourite_salmon.nodes.find { |node| node.name == 'activity:verb' }
-      expect(verb.text).to eq OStatus::TagManager::VERBS[:favorite]
-    end
-
-    it 'appends activity:object element with status' do
-      status = Fabricate(:status, created_at: '2000-01-01T00:00:00Z')
-      favourite = Fabricate(:favourite, status: status)
-
-      favourite_salmon = OStatus::AtomSerializer.new.favourite_salmon(favourite)
-
-      object = favourite_salmon.nodes.find { |node| node.name == 'activity:object' }
-      expect(object.id.text).to eq "https://cb6e6126.ngrok.io/users/#{status.account.to_param}/statuses/#{status.id}"
-    end
-
-    it 'appends thr:in-reply-to element for status' do
-      status_account = Fabricate(:account, username: 'username')
-      status = Fabricate(:status, account: status_account, created_at: '2000-01-01T00:00:00Z')
-      favourite = Fabricate(:favourite, status: status)
-
-      favourite_salmon = OStatus::AtomSerializer.new.favourite_salmon(favourite)
-
-      in_reply_to = favourite_salmon.nodes.find { |node| node.name == 'thr:in-reply-to' }
-      expect(in_reply_to.ref).to eq "https://cb6e6126.ngrok.io/users/#{status.account.to_param}/statuses/#{status.id}"
-      expect(in_reply_to.href).to eq "https://cb6e6126.ngrok.io/@username/#{status.id}"
-    end
-
-    it 'includes description' do
-      account = Fabricate(:account, domain: nil, username: 'account')
-      status_account = Fabricate(:account, domain: 'remote', username: 'status_account')
-      status = Fabricate(:status, account: status_account)
-      favourite = Fabricate(:favourite, account: account, status: status)
-
-      favourite_salmon = OStatus::AtomSerializer.new.favourite_salmon(favourite)
-
-      expect(favourite_salmon.title.text).to eq 'account favourited a status by status_account@remote'
-      expect(favourite_salmon.content.text).to eq 'account favourited a status by status_account@remote'
-    end
-  end
-
-  describe '#unfavourite_salmon' do
-    include_examples 'namespaces' do
-      def serialize
-        favourite = Fabricate(:favourite)
-        OStatus::AtomSerializer.new.favourite_salmon(favourite)
-      end
-    end
-
-    it 'returns entry element' do
-      favourite = Fabricate(:favourite)
-      unfavourite_salmon = OStatus::AtomSerializer.new.unfavourite_salmon(favourite)
-      expect(unfavourite_salmon.name).to eq 'entry'
-    end
-
-    it 'appends id element with unique tag' do
-      favourite = Fabricate(:favourite)
-
-      time_before = Time.zone.now
-      unfavourite_salmon = OStatus::AtomSerializer.new.unfavourite_salmon(favourite)
-      time_after = Time.zone.now
-
-      expect(unfavourite_salmon.id.text).to(
-        eq(OStatus::TagManager.instance.unique_tag(time_before.utc, favourite.id, 'Favourite'))
-          .or(eq(OStatus::TagManager.instance.unique_tag(time_after.utc, favourite.id, 'Favourite')))
-      )
-    end
-
-    it 'appends author element with account' do
-      account = Fabricate(:account, domain: nil, username: 'username')
-      favourite = Fabricate(:favourite, account: account)
-
-      unfavourite_salmon = OStatus::AtomSerializer.new.unfavourite_salmon(favourite)
-
-      expect(unfavourite_salmon.author.id.text).to eq 'https://cb6e6126.ngrok.io/users/username'
-    end
-
-    it 'appends activity:object-type element with activity type' do
-      favourite = Fabricate(:favourite)
-
-      unfavourite_salmon = OStatus::AtomSerializer.new.unfavourite_salmon(favourite)
-
-      object_type = unfavourite_salmon.nodes.find { |node| node.name == 'activity:object-type' }
-      expect(object_type.text).to eq 'http://activitystrea.ms/schema/1.0/activity'
-    end
-
-    it 'appends activity:verb element with favorite' do
-      favourite = Fabricate(:favourite)
-
-      unfavourite_salmon = OStatus::AtomSerializer.new.unfavourite_salmon(favourite)
-
-      verb = unfavourite_salmon.nodes.find { |node| node.name == 'activity:verb' }
-      expect(verb.text).to eq OStatus::TagManager::VERBS[:unfavorite]
-    end
-
-    it 'appends activity:object element with status' do
-      status = Fabricate(:status, created_at: '2000-01-01T00:00:00Z')
-      favourite = Fabricate(:favourite, status: status)
-
-      unfavourite_salmon = OStatus::AtomSerializer.new.unfavourite_salmon(favourite)
-
-      object = unfavourite_salmon.nodes.find { |node| node.name == 'activity:object' }
-      expect(object.id.text).to eq "https://cb6e6126.ngrok.io/users/#{status.account.to_param}/statuses/#{status.id}"
-    end
-
-    it 'appends thr:in-reply-to element for status' do
-      status_account = Fabricate(:account, username: 'username')
-      status = Fabricate(:status, account: status_account, created_at: '2000-01-01T00:00:00Z')
-      favourite = Fabricate(:favourite, status: status)
-
-      unfavourite_salmon = OStatus::AtomSerializer.new.unfavourite_salmon(favourite)
-
-      in_reply_to = unfavourite_salmon.nodes.find { |node| node.name == 'thr:in-reply-to' }
-      expect(in_reply_to.ref).to eq "https://cb6e6126.ngrok.io/users/#{status.account.to_param}/statuses/#{status.id}"
-      expect(in_reply_to.href).to eq "https://cb6e6126.ngrok.io/@username/#{status.id}"
-    end
-
-    it 'includes description' do
-      account = Fabricate(:account, domain: nil, username: 'account')
-      status_account = Fabricate(:account, domain: 'remote', username: 'status_account')
-      status = Fabricate(:status, account: status_account)
-      favourite = Fabricate(:favourite, account: account, status: status)
-
-      unfavourite_salmon = OStatus::AtomSerializer.new.unfavourite_salmon(favourite)
-
-      expect(unfavourite_salmon.title.text).to eq 'account no longer favourites a status by status_account@remote'
-      expect(unfavourite_salmon.content.text).to eq 'account no longer favourites a status by status_account@remote'
-    end
-  end
-
-  describe '#follow_salmon' do
-    include_examples 'namespaces' do
-      def serialize
-        follow = Fabricate(:follow)
-        OStatus::AtomSerializer.new.follow_salmon(follow)
-      end
-    end
-
-    it 'returns entry element' do
-      follow = Fabricate(:follow)
-      follow_salmon = OStatus::AtomSerializer.new.follow_salmon(follow)
-      expect(follow_salmon.name).to eq 'entry'
-    end
-
-    it 'appends id element with unique tag' do
-      follow = Fabricate(:follow, created_at: '2000-01-01T00:00:00Z')
-      follow_salmon = OStatus::AtomSerializer.new.follow_salmon(follow)
-      expect(follow_salmon.id.text).to eq "tag:cb6e6126.ngrok.io,2000-01-01:objectId=#{follow.id}:objectType=Follow"
-    end
-
-    it 'appends author element with account' do
-      account = Fabricate(:account, domain: nil, username: 'username')
-      follow = Fabricate(:follow, account: account)
-
-      follow_salmon = OStatus::AtomSerializer.new.follow_salmon(follow)
-
-      expect(follow_salmon.author.id.text).to eq 'https://cb6e6126.ngrok.io/users/username'
-    end
-
-    it 'appends activity:object-type element with activity type' do
-      follow = Fabricate(:follow)
-
-      follow_salmon = OStatus::AtomSerializer.new.follow_salmon(follow)
-
-      object_type = follow_salmon.nodes.find { |node| node.name == 'activity:object-type' }
-      expect(object_type.text).to eq OStatus::TagManager::TYPES[:activity]
-    end
-
-    it 'appends activity:verb element with follow' do
-      follow = Fabricate(:follow)
-
-      follow_salmon = OStatus::AtomSerializer.new.follow_salmon(follow)
-
-      verb = follow_salmon.nodes.find { |node| node.name == 'activity:verb' }
-      expect(verb.text).to eq OStatus::TagManager::VERBS[:follow]
-    end
-
-    it 'appends activity:object element with target account' do
-      target_account = Fabricate(:account, domain: 'domain.test', uri: 'https://domain.test/id')
-      follow = Fabricate(:follow, target_account: target_account)
-
-      follow_salmon = OStatus::AtomSerializer.new.follow_salmon(follow)
-
-      object = follow_salmon.nodes.find { |node| node.name == 'activity:object' }
-      expect(object.id.text).to eq 'https://domain.test/id'
-    end
-
-    it 'includes description' do
-      account = Fabricate(:account, domain: nil, username: 'account')
-      target_account = Fabricate(:account, domain: 'remote', username: 'target_account')
-      follow = Fabricate(:follow, account: account, target_account: target_account)
-
-      follow_salmon = OStatus::AtomSerializer.new.follow_salmon(follow)
-
-      expect(follow_salmon.title.text).to eq 'account started following target_account@remote'
-      expect(follow_salmon.content.text).to eq 'account started following target_account@remote'
-    end
-  end
-
-  describe '#unfollow_salmon' do
-    include_examples 'namespaces' do
-      def serialize
-        follow = Fabricate(:follow)
-        follow.destroy!
-        OStatus::AtomSerializer.new.unfollow_salmon(follow)
-      end
-    end
-
-    it 'returns entry element' do
-      follow = Fabricate(:follow)
-      follow.destroy!
-
-      unfollow_salmon = OStatus::AtomSerializer.new.unfollow_salmon(follow)
-
-      expect(unfollow_salmon.name).to eq 'entry'
-    end
-
-    it 'appends id element with unique tag' do
-      follow = Fabricate(:follow)
-      follow.destroy!
-
-      time_before = Time.zone.now
-      unfollow_salmon = OStatus::AtomSerializer.new.unfollow_salmon(follow)
-      time_after = Time.zone.now
-
-      expect(unfollow_salmon.id.text).to(
-        eq(OStatus::TagManager.instance.unique_tag(time_before.utc, follow.id, 'Follow'))
-          .or(eq(OStatus::TagManager.instance.unique_tag(time_after.utc, follow.id, 'Follow')))
-      )
-    end
-
-    it 'appends title element with description' do
-      account = Fabricate(:account, domain: nil, username: 'account')
-      target_account = Fabricate(:account, domain: 'remote', username: 'target_account')
-      follow = Fabricate(:follow, account: account, target_account: target_account)
-      follow.destroy!
-
-      unfollow_salmon = OStatus::AtomSerializer.new.unfollow_salmon(follow)
-
-      expect(unfollow_salmon.title.text).to eq 'account is no longer following target_account@remote'
-    end
-
-    it 'appends content element with description' do
-      account = Fabricate(:account, domain: nil, username: 'account')
-      target_account = Fabricate(:account, domain: 'remote', username: 'target_account')
-      follow = Fabricate(:follow, account: account, target_account: target_account)
-      follow.destroy!
-
-      unfollow_salmon = OStatus::AtomSerializer.new.unfollow_salmon(follow)
-
-      expect(unfollow_salmon.content.text).to eq 'account is no longer following target_account@remote'
-    end
-
-    it 'appends author element with account' do
-      account = Fabricate(:account, domain: nil, username: 'username')
-      follow = Fabricate(:follow, account: account)
-      follow.destroy!
-
-      unfollow_salmon = OStatus::AtomSerializer.new.unfollow_salmon(follow)
-
-      expect(unfollow_salmon.author.id.text).to eq 'https://cb6e6126.ngrok.io/users/username'
-    end
-
-    it 'appends activity:object-type element with activity type' do
-      follow = Fabricate(:follow)
-      follow.destroy!
-
-      unfollow_salmon = OStatus::AtomSerializer.new.unfollow_salmon(follow)
-
-      object_type = unfollow_salmon.nodes.find { |node| node.name == 'activity:object-type' }
-      expect(object_type.text).to eq OStatus::TagManager::TYPES[:activity]
-    end
-
-    it 'appends activity:verb element with follow' do
-      follow = Fabricate(:follow)
-      follow.destroy!
-
-      unfollow_salmon = OStatus::AtomSerializer.new.unfollow_salmon(follow)
-
-      verb = unfollow_salmon.nodes.find { |node| node.name == 'activity:verb' }
-      expect(verb.text).to eq OStatus::TagManager::VERBS[:unfollow]
-    end
-
-    it 'appends activity:object element with target account' do
-      target_account = Fabricate(:account, domain: 'domain.test', uri: 'https://domain.test/id')
-      follow = Fabricate(:follow, target_account: target_account)
-      follow.destroy!
-
-      unfollow_salmon = OStatus::AtomSerializer.new.unfollow_salmon(follow)
-
-      object = unfollow_salmon.nodes.find { |node| node.name == 'activity:object' }
-      expect(object.id.text).to eq 'https://domain.test/id'
-    end
-  end
-
-  describe '#follow_request_salmon' do
-    include_examples 'namespaces' do
-      def serialize
-        follow_request = Fabricate(:follow_request)
-        OStatus::AtomSerializer.new.follow_request_salmon(follow_request)
-      end
-    end
-
-    context do
-      def serialize(follow_request)
-        OStatus::AtomSerializer.new.follow_request_salmon(follow_request)
-      end
-
-      it_behaves_like 'follow request salmon'
-
-      it 'appends id element with unique tag' do
-        follow_request = Fabricate(:follow_request, created_at: '2000-01-01T00:00:00Z')
-        follow_request_salmon = serialize(follow_request)
-        expect(follow_request_salmon.id.text).to eq "tag:cb6e6126.ngrok.io,2000-01-01:objectId=#{follow_request.id}:objectType=FollowRequest"
-      end
-
-      it 'appends title element with description' do
-        account = Fabricate(:account, domain: nil, username: 'account')
-        target_account = Fabricate(:account, domain: 'remote', username: 'target_account')
-        follow_request = Fabricate(:follow_request, account: account, target_account: target_account)
-        follow_request_salmon = serialize(follow_request)
-        expect(follow_request_salmon.title.text).to eq 'account requested to follow target_account@remote'
-      end
-    end
-  end
-
-  describe '#authorize_follow_request_salmon' do
-    include_examples 'namespaces' do
-      def serialize
-        follow_request = Fabricate(:follow_request)
-        OStatus::AtomSerializer.new.authorize_follow_request_salmon(follow_request)
-      end
-    end
-
-    it_behaves_like 'follow request salmon' do
-      def serialize(follow_request)
-        authorize_follow_request_salmon = OStatus::AtomSerializer.new.authorize_follow_request_salmon(follow_request)
-        authorize_follow_request_salmon.nodes.find { |node| node.name == 'activity:object' }
-      end
-    end
-
-    it 'appends id element with unique tag' do
-      follow_request = Fabricate(:follow_request)
-
-      time_before = Time.zone.now
-      authorize_follow_request_salmon = OStatus::AtomSerializer.new.authorize_follow_request_salmon(follow_request)
-      time_after = Time.zone.now
-
-      expect(authorize_follow_request_salmon.id.text).to(
-        eq(OStatus::TagManager.instance.unique_tag(time_before.utc, follow_request.id, 'FollowRequest'))
-          .or(eq(OStatus::TagManager.instance.unique_tag(time_after.utc, follow_request.id, 'FollowRequest')))
-      )
-    end
-
-    it 'appends title element with description' do
-      account = Fabricate(:account, domain: 'remote', username: 'account')
-      target_account = Fabricate(:account, domain: nil, username: 'target_account')
-      follow_request = Fabricate(:follow_request, account: account, target_account: target_account)
-
-      authorize_follow_request_salmon = OStatus::AtomSerializer.new.authorize_follow_request_salmon(follow_request)
-
-      expect(authorize_follow_request_salmon.title.text).to eq 'target_account authorizes follow request by account@remote'
-    end
-
-    it 'appends activity:object-type element with activity type' do
-      follow_request = Fabricate(:follow_request)
-
-      authorize_follow_request_salmon = OStatus::AtomSerializer.new.authorize_follow_request_salmon(follow_request)
-
-      object_type = authorize_follow_request_salmon.nodes.find { |node| node.name == 'activity:object-type' }
-      expect(object_type.text).to eq OStatus::TagManager::TYPES[:activity]
-    end
-
-    it 'appends activity:verb element with authorize' do
-      follow_request = Fabricate(:follow_request)
-
-      authorize_follow_request_salmon = OStatus::AtomSerializer.new.authorize_follow_request_salmon(follow_request)
-
-      verb = authorize_follow_request_salmon.nodes.find { |node| node.name == 'activity:verb' }
-      expect(verb.text).to eq OStatus::TagManager::VERBS[:authorize]
-    end
-  end
-
-  describe '#reject_follow_request_salmon' do
-    include_examples 'namespaces' do
-      def serialize
-        follow_request = Fabricate(:follow_request)
-        OStatus::AtomSerializer.new.reject_follow_request_salmon(follow_request)
-      end
-    end
-
-    it_behaves_like 'follow request salmon' do
-      def serialize(follow_request)
-        reject_follow_request_salmon = OStatus::AtomSerializer.new.reject_follow_request_salmon(follow_request)
-        reject_follow_request_salmon.nodes.find { |node| node.name == 'activity:object' }
-      end
-    end
-
-    it 'appends id element with unique tag' do
-      follow_request = Fabricate(:follow_request)
-
-      time_before = Time.zone.now
-      reject_follow_request_salmon = OStatus::AtomSerializer.new.reject_follow_request_salmon(follow_request)
-      time_after = Time.zone.now
-
-      expect(reject_follow_request_salmon.id.text).to(
-        eq(OStatus::TagManager.instance.unique_tag(time_before.utc, follow_request.id, 'FollowRequest'))
-          .or(OStatus::TagManager.instance.unique_tag(time_after.utc, follow_request.id, 'FollowRequest'))
-      )
-    end
-
-    it 'appends title element with description' do
-      account = Fabricate(:account, domain: 'remote', username: 'account')
-      target_account = Fabricate(:account, domain: nil, username: 'target_account')
-      follow_request = Fabricate(:follow_request, account: account, target_account: target_account)
-      reject_follow_request_salmon = OStatus::AtomSerializer.new.reject_follow_request_salmon(follow_request)
-      expect(reject_follow_request_salmon.title.text).to eq 'target_account rejects follow request by account@remote'
-    end
-
-    it 'appends activity:object-type element with activity type' do
-      follow_request = Fabricate(:follow_request)
-      reject_follow_request_salmon = OStatus::AtomSerializer.new.reject_follow_request_salmon(follow_request)
-      object_type = reject_follow_request_salmon.nodes.find { |node| node.name == 'activity:object-type' }
-      expect(object_type.text).to eq OStatus::TagManager::TYPES[:activity]
-    end
-
-    it 'appends activity:verb element with authorize' do
-      follow_request = Fabricate(:follow_request)
-      reject_follow_request_salmon = OStatus::AtomSerializer.new.reject_follow_request_salmon(follow_request)
-      verb = reject_follow_request_salmon.nodes.find { |node| node.name == 'activity:verb' }
-      expect(verb.text).to eq OStatus::TagManager::VERBS[:reject]
-    end
-  end
-
-  describe '#object' do
-    include_examples 'status attributes' do
-      def serialize(status)
-        OStatus::AtomSerializer.new.object(status)
-      end
-    end
-
-    it 'returns activity:object element' do
-      status = Fabricate(:status)
-      object = OStatus::AtomSerializer.new.object(status)
-      expect(object.name).to eq 'activity:object'
-    end
-
-    it 'appends id element with URL for status' do
-      status = Fabricate(:status, created_at: '2000-01-01T00:00:00Z')
-      object = OStatus::AtomSerializer.new.object(status)
-      expect(object.id.text).to eq "https://cb6e6126.ngrok.io/users/#{status.account.to_param}/statuses/#{status.id}"
-    end
-
-    it 'appends published element with created date' do
-      status = Fabricate(:status, created_at: '2000-01-01T00:00:00Z')
-      object = OStatus::AtomSerializer.new.object(status)
-      expect(object.published.text).to eq '2000-01-01T00:00:00Z'
-    end
-
-    it 'appends updated element with updated date' do
-      status = Fabricate(:status)
-      status.updated_at = '2000-01-01T00:00:00Z'
-      object = OStatus::AtomSerializer.new.object(status)
-      expect(object.updated.text).to eq '2000-01-01T00:00:00Z'
-    end
-
-    it 'appends title element with title' do
-      account = Fabricate(:account, username: 'username')
-      status = Fabricate(:status, account: account)
-
-      object = OStatus::AtomSerializer.new.object(status)
-
-      expect(object.title.text).to eq 'New status by username'
-    end
-
-    it 'appends author element with account' do
-      account = Fabricate(:account, username: 'username')
-      status = Fabricate(:status, account: account)
-
-      entry = OStatus::AtomSerializer.new.object(status)
-
-      expect(entry.author.id.text).to eq 'https://cb6e6126.ngrok.io/users/username'
-    end
-
-    it 'appends activity:object-type element with object type' do
-      status = Fabricate(:status)
-
-      entry = OStatus::AtomSerializer.new.object(status)
-
-      object_type = entry.nodes.find { |node| node.name == 'activity:object-type' }
-      expect(object_type.text).to eq OStatus::TagManager::TYPES[:note]
-    end
-
-    it 'appends activity:verb element with verb' do
-      status = Fabricate(:status)
-
-      entry = OStatus::AtomSerializer.new.object(status)
-
-      object_type = entry.nodes.find { |node| node.name == 'activity:verb' }
-      expect(object_type.text).to eq OStatus::TagManager::VERBS[:post]
-    end
-
-    it 'appends link element for an alternative' do
-      account = Fabricate(:account, username: 'username')
-      status = Fabricate(:status, account: account)
-
-      entry = OStatus::AtomSerializer.new.object(status)
-
-      link = entry.nodes.find { |node| node.name == 'link' && node[:rel] == 'alternate' && node[:type] == 'text/html' }
-      expect(link[:type]).to eq 'text/html'
-      expect(link[:href]).to eq "https://cb6e6126.ngrok.io/@username/#{status.id}"
-    end
-
-    it 'appends thr:in-reply-to element if it is a reply and thread is not nil' do
-      account = Fabricate(:account, username: 'username')
-      thread = Fabricate(:status, account: account, created_at: '2000-01-01T00:00:00Z')
-      reply = Fabricate(:status, thread: thread)
-
-      entry = OStatus::AtomSerializer.new.object(reply)
-
-      in_reply_to = entry.nodes.find { |node| node.name == 'thr:in-reply-to' }
-      expect(in_reply_to.ref).to eq "https://cb6e6126.ngrok.io/users/#{thread.account.to_param}/statuses/#{thread.id}"
-      expect(in_reply_to.href).to eq "https://cb6e6126.ngrok.io/@username/#{thread.id}"
-    end
-
-    it 'does not append thr:in-reply-to element if thread is nil' do
-      status = Fabricate(:status, thread: nil)
-      entry = OStatus::AtomSerializer.new.object(status)
-      entry.nodes.each { |node| expect(node.name).not_to eq 'thr:in-reply-to' }
-    end
-
-    it 'does not append ostatus:conversation element if conversation_id is nil' do
-      status = Fabricate.build(:status, conversation_id: nil)
-      status.save!(validate: false)
-
-      entry = OStatus::AtomSerializer.new.object(status)
-
-      entry.nodes.each { |node| expect(node.name).not_to eq 'ostatus:conversation' }
-    end
-
-    it 'appends ostatus:conversation element if conversation_id is not nil' do
-      status = Fabricate(:status)
-      status.conversation.update!(created_at: '2000-01-01T00:00:00Z')
-
-      entry = OStatus::AtomSerializer.new.object(status)
-
-      conversation = entry.nodes.find { |node| node.name == 'ostatus:conversation' }
-      expect(conversation[:ref]).to eq "tag:cb6e6126.ngrok.io,2000-01-01:objectId=#{status.conversation.id}:objectType=Conversation"
-    end
-  end
-end
diff --git a/spec/lib/status_finder_spec.rb b/spec/lib/status_finder_spec.rb
index 6b4ee434f..61483f4bf 100644
--- a/spec/lib/status_finder_spec.rb
+++ b/spec/lib/status_finder_spec.rb
@@ -25,15 +25,6 @@ describe StatusFinder do
       end
     end
 
-    context 'with a stream entry url' do
-      let(:stream_entry) { Fabricate(:stream_entry) }
-      let(:url) { account_stream_entry_url(stream_entry.account, stream_entry) }
-
-      it 'finds the stream entry' do
-        expect(subject.status).to eq(stream_entry.status)
-      end
-    end
-
     context 'with a remote url even if id exists on local' do
       let(:status) { Fabricate(:status) }
       let(:url) { "https://example.com/users/test/statuses/#{status.id}" }
diff --git a/spec/lib/tag_manager_spec.rb b/spec/lib/tag_manager_spec.rb
index 3a804ac0f..e9a7aa934 100644
--- a/spec/lib/tag_manager_spec.rb
+++ b/spec/lib/tag_manager_spec.rb
@@ -119,46 +119,4 @@ RSpec.describe TagManager do
       expect(TagManager.instance.same_acct?('username', 'incorrect@Cb6E6126.nGrOk.Io')).to eq false
     end
   end
-
-  describe '#url_for' do
-    let(:alice) { Fabricate(:account, username: 'alice') }
-
-    subject { TagManager.instance.url_for(target) }
-
-    context 'activity object' do
-      let(:target) { Fabricate(:status, account: alice, reblog: Fabricate(:status)).stream_entry }
-
-      it 'returns the unique tag for status' do
-        expect(target.object_type).to eq :activity
-        is_expected.to eq "https://cb6e6126.ngrok.io/@alice/#{target.id}"
-      end
-    end
-
-    context 'comment object' do
-      let(:target) { Fabricate(:status, account: alice, reply: true) }
-
-      it 'returns the unique tag for status' do
-        expect(target.object_type).to eq :comment
-        is_expected.to eq "https://cb6e6126.ngrok.io/@alice/#{target.id}"
-      end
-    end
-
-    context 'note object' do
-      let(:target) { Fabricate(:status, account: alice, reply: false, thread: nil) }
-
-      it 'returns the unique tag for status' do
-        expect(target.object_type).to eq :note
-        is_expected.to eq "https://cb6e6126.ngrok.io/@alice/#{target.id}"
-      end
-    end
-
-    context 'person object' do
-      let(:target) { alice }
-
-      it 'returns the URL for account' do
-        expect(target.object_type).to eq :person
-        is_expected.to eq 'https://cb6e6126.ngrok.io/@alice'
-      end
-    end
-  end
 end
diff --git a/spec/models/concerns/streamable_spec.rb b/spec/models/concerns/streamable_spec.rb
deleted file mode 100644
index b5f2d5192..000000000
--- a/spec/models/concerns/streamable_spec.rb
+++ /dev/null
@@ -1,63 +0,0 @@
-# frozen_string_literal: true
-
-require 'rails_helper'
-
-RSpec.describe Streamable do
-  class Parent
-    def title; end
-
-    def target; end
-
-    def thread; end
-
-    def self.has_one(*); end
-
-    def self.after_create; end
-  end
-
-  class Child < Parent
-    include Streamable
-  end
-
-  child = Child.new
-
-  describe '#title' do
-    it 'calls Parent#title' do
-      expect_any_instance_of(Parent).to receive(:title)
-      child.title
-    end
-  end
-
-  describe '#content' do
-    it 'calls #title' do
-      expect_any_instance_of(Parent).to receive(:title)
-      child.content
-    end
-  end
-
-  describe '#target' do
-    it 'calls Parent#target' do
-      expect_any_instance_of(Parent).to receive(:target)
-      child.target
-    end
-  end
-
-  describe '#object_type' do
-    it 'returns :activity' do
-      expect(child.object_type).to eq :activity
-    end
-  end
-
-  describe '#thread' do
-    it 'calls Parent#thread' do
-      expect_any_instance_of(Parent).to receive(:thread)
-      child.thread
-    end
-  end
-
-  describe '#hidden?' do
-    it 'returns false' do
-      expect(child.hidden?).to be false
-    end
-  end
-end
diff --git a/spec/models/remote_profile_spec.rb b/spec/models/remote_profile_spec.rb
deleted file mode 100644
index da5048f0a..000000000
--- a/spec/models/remote_profile_spec.rb
+++ /dev/null
@@ -1,143 +0,0 @@
-# frozen_string_literal: true
-
-require 'rails_helper'
-
-RSpec.describe RemoteProfile do
-  let(:remote_profile) { RemoteProfile.new(body) }
-  let(:body) do
-    <<-XML
-      <feed xmlns="http://www.w3.org/2005/Atom">
-      <author>John</author>
-    XML
-  end
-
-  describe '.initialize' do
-    it 'calls Nokogiri::XML.parse' do
-      expect(Nokogiri::XML).to receive(:parse).with(body, nil, 'utf-8')
-      RemoteProfile.new(body)
-    end
-
-    it 'sets document' do
-      remote_profile = RemoteProfile.new(body)
-      expect(remote_profile).not_to be nil
-    end
-  end
-
-  describe '#root' do
-    let(:document) { remote_profile.document }
-
-    it 'callse document.at_xpath' do
-      expect(document).to receive(:at_xpath).with(
-        '/atom:feed|/atom:entry',
-        atom: OStatus::TagManager::XMLNS
-      )
-
-      remote_profile.root
-    end
-  end
-
-  describe '#author' do
-    let(:root) { remote_profile.root }
-
-    it 'calls root.at_xpath' do
-      expect(root).to receive(:at_xpath).with(
-        './atom:author|./dfrn:owner',
-        atom: OStatus::TagManager::XMLNS,
-        dfrn: OStatus::TagManager::DFRN_XMLNS
-      )
-
-      remote_profile.author
-    end
-  end
-
-  describe '#hub_link' do
-    let(:root) { remote_profile.root }
-
-    it 'calls #link_href_from_xml' do
-      expect(remote_profile).to receive(:link_href_from_xml).with(root, 'hub')
-      remote_profile.hub_link
-    end
-  end
-
-  describe '#display_name' do
-    let(:author) { remote_profile.author }
-
-    it 'calls author.at_xpath.content' do
-      expect(author).to receive_message_chain(:at_xpath, :content).with(
-        './poco:displayName',
-        poco: OStatus::TagManager::POCO_XMLNS
-      ).with(no_args)
-
-      remote_profile.display_name
-    end
-  end
-
-  describe '#note' do
-    let(:author) { remote_profile.author }
-
-    it 'calls author.at_xpath.content' do
-      expect(author).to receive_message_chain(:at_xpath, :content).with(
-        './atom:summary|./poco:note',
-        atom: OStatus::TagManager::XMLNS,
-        poco: OStatus::TagManager::POCO_XMLNS
-      ).with(no_args)
-
-      remote_profile.note
-    end
-  end
-
-  describe '#scope' do
-    let(:author) { remote_profile.author }
-
-    it 'calls author.at_xpath.content' do
-      expect(author).to receive_message_chain(:at_xpath, :content).with(
-        './mastodon:scope',
-        mastodon: OStatus::TagManager::MTDN_XMLNS
-      ).with(no_args)
-
-      remote_profile.scope
-    end
-  end
-
-  describe '#avatar' do
-    let(:author) { remote_profile.author }
-
-    it 'calls #link_href_from_xml' do
-      expect(remote_profile).to receive(:link_href_from_xml).with(author, 'avatar')
-      remote_profile.avatar
-    end
-  end
-
-  describe '#header' do
-    let(:author) { remote_profile.author }
-
-    it 'calls #link_href_from_xml' do
-      expect(remote_profile).to receive(:link_href_from_xml).with(author, 'header')
-      remote_profile.header
-    end
-  end
-
-  describe '#locked?' do
-    before do
-      allow(remote_profile).to receive(:scope).and_return(scope)
-    end
-
-    subject { remote_profile.locked? }
-
-    context 'scope is private' do
-      let(:scope) { 'private' }
-
-      it 'returns true' do
-        is_expected.to be true
-      end
-    end
-
-    context 'scope is not private' do
-      let(:scope) { 'public' }
-
-      it 'returns false' do
-        is_expected.to be false
-      end
-    end
-  end
-end
diff --git a/spec/models/stream_entry_spec.rb b/spec/models/stream_entry_spec.rb
deleted file mode 100644
index 8f8bfbd58..000000000
--- a/spec/models/stream_entry_spec.rb
+++ /dev/null
@@ -1,192 +0,0 @@
-require 'rails_helper'
-
-RSpec.describe StreamEntry, type: :model do
-  let(:alice)     { Fabricate(:account, username: 'alice') }
-  let(:bob)       { Fabricate(:account, username: 'bob') }
-  let(:status)    { Fabricate(:status, account: alice) }
-  let(:reblog)    { Fabricate(:status, account: bob, reblog: status) }
-  let(:reply)     { Fabricate(:status, account: bob, thread: status) }
-  let(:stream_entry) { Fabricate(:stream_entry, activity: activity) }
-  let(:activity)     { reblog }
-
-  describe '#object_type' do
-    before do
-      allow(stream_entry).to receive(:orphaned?).and_return(orphaned)
-      allow(stream_entry).to receive(:targeted?).and_return(targeted)
-    end
-
-    subject { stream_entry.object_type }
-
-    context 'orphaned? is true' do
-      let(:orphaned) { true }
-      let(:targeted) { false }
-
-      it 'returns :activity' do
-        is_expected.to be :activity
-      end
-    end
-
-    context 'targeted? is true' do
-      let(:orphaned) { false }
-      let(:targeted) { true }
-
-      it 'returns :activity' do
-        is_expected.to be :activity
-      end
-    end
-
-    context 'orphaned? and targeted? are false' do
-      let(:orphaned) { false }
-      let(:targeted) { false }
-
-      context 'activity is reblog' do
-        let(:activity) { reblog }
-
-        it 'returns :note' do
-          is_expected.to be :note
-        end
-      end
-
-      context 'activity is reply' do
-        let(:activity) { reply }
-
-        it 'returns :comment' do
-          is_expected.to be :comment
-        end
-      end
-    end
-  end
-
-  describe '#verb' do
-    before do
-      allow(stream_entry).to receive(:orphaned?).and_return(orphaned)
-    end
-
-    subject { stream_entry.verb }
-
-    context 'orphaned? is true' do
-      let(:orphaned) { true }
-
-      it 'returns :delete' do
-        is_expected.to be :delete
-      end
-    end
-
-    context 'orphaned? is false' do
-      let(:orphaned) { false }
-
-      context 'activity is reblog' do
-        let(:activity) { reblog }
-
-        it 'returns :share' do
-          is_expected.to be :share
-        end
-      end
-
-      context 'activity is reply' do
-        let(:activity) { reply }
-
-        it 'returns :post' do
-          is_expected.to be :post
-        end
-      end
-    end
-  end
-
-  describe '#mentions' do
-    before do
-      allow(stream_entry).to receive(:orphaned?).and_return(orphaned)
-    end
-
-    subject { stream_entry.mentions }
-
-    context 'orphaned? is true' do
-      let(:orphaned) { true }
-
-      it 'returns []' do
-        is_expected.to eq []
-      end
-    end
-
-    context 'orphaned? is false' do
-      before do
-        reblog.mentions << Fabricate(:mention, account: alice)
-        reblog.mentions << Fabricate(:mention, account: bob)
-      end
-
-      let(:orphaned) { false }
-
-      it 'returns [Account] includes alice and bob' do
-        is_expected.to eq [alice, bob]
-      end
-    end
-  end
-
-  describe '#targeted?' do
-    it 'returns true for a reblog' do
-      expect(reblog.stream_entry.targeted?).to be true
-    end
-
-    it 'returns false otherwise' do
-      expect(status.stream_entry.targeted?).to be false
-    end
-  end
-
-  describe '#threaded?' do
-    it 'returns true for a reply' do
-      expect(reply.stream_entry.threaded?).to be true
-    end
-
-    it 'returns false otherwise' do
-      expect(status.stream_entry.threaded?).to be false
-    end
-  end
-
-  describe 'delegated methods' do
-    context 'with a nil status' do
-      subject { described_class.new(status: nil) }
-
-      it 'returns nil for target' do
-        expect(subject.target).to be_nil
-      end
-
-      it 'returns nil for title' do
-        expect(subject.title).to be_nil
-      end
-
-      it 'returns nil for content' do
-        expect(subject.content).to be_nil
-      end
-
-      it 'returns nil for thread' do
-        expect(subject.thread).to be_nil
-      end
-    end
-
-    context 'with a real status' do
-      let(:original) { Fabricate(:status, text: 'Test status') }
-      let(:status) { Fabricate(:status, reblog: original, thread: original) }
-      subject { described_class.new(status: status) }
-
-      it 'delegates target' do
-        expect(status.target).not_to be_nil
-        expect(subject.target).to eq(status.target)
-      end
-
-      it 'delegates title' do
-        expect(status.title).not_to be_nil
-        expect(subject.title).to eq(status.title)
-      end
-
-      it 'delegates content' do
-        expect(status.content).not_to be_nil
-        expect(subject.content).to eq(status.content)
-      end
-
-      it 'delegates thread' do
-        expect(status.thread).not_to be_nil
-        expect(subject.thread).to eq(status.thread)
-      end
-    end
-  end
-end
diff --git a/spec/services/process_mentions_service_spec.rb b/spec/services/process_mentions_service_spec.rb
index 35a804f2b..b1abd79b0 100644
--- a/spec/services/process_mentions_service_spec.rb
+++ b/spec/services/process_mentions_service_spec.rb
@@ -15,8 +15,8 @@ RSpec.describe ProcessMentionsService, type: :service do
       subject.call(status)
     end
 
-    it 'creates a mention' do
-      expect(remote_user.mentions.where(status: status).count).to eq 1
+    it 'does not create a mention' do
+      expect(remote_user.mentions.where(status: status).count).to eq 0
     end
   end
 
diff --git a/spec/services/suspend_account_service_spec.rb b/spec/services/suspend_account_service_spec.rb
index 6f45762aa..896ac17a3 100644
--- a/spec/services/suspend_account_service_spec.rb
+++ b/spec/services/suspend_account_service_spec.rb
@@ -27,14 +27,13 @@ RSpec.describe SuspendAccountService, type: :service do
         [
           account.statuses,
           account.media_attachments,
-          account.stream_entries,
           account.notifications,
           account.favourites,
           account.active_relationships,
           account.passive_relationships,
           account.subscriptions
         ].map(&:count)
-      }.from([1, 1, 1, 1, 1, 1, 1, 1]).to([0, 0, 0, 0, 0, 0, 0, 0])
+      }.from([1, 1, 1, 1, 1, 1, 1]).to([0, 0, 0, 0, 0, 0, 0])
     end
 
     it 'sends a delete actor activity to all known inboxes' do
@@ -70,14 +69,13 @@ RSpec.describe SuspendAccountService, type: :service do
         [
           remote_bob.statuses,
           remote_bob.media_attachments,
-          remote_bob.stream_entries,
           remote_bob.notifications,
           remote_bob.favourites,
           remote_bob.active_relationships,
           remote_bob.passive_relationships,
           remote_bob.subscriptions
         ].map(&:count)
-      }.from([1, 1, 1, 1, 1, 1, 1, 1]).to([0, 0, 0, 0, 0, 0, 0, 0])
+      }.from([1, 1, 1, 1, 1, 1, 1]).to([0, 0, 0, 0, 0, 0, 0])
     end
 
     it 'sends a reject follow to follwer inboxes' do
diff --git a/spec/views/statuses/show.html.haml_spec.rb b/spec/views/statuses/show.html.haml_spec.rb
new file mode 100644
index 000000000..dbda3b665
--- /dev/null
+++ b/spec/views/statuses/show.html.haml_spec.rb
@@ -0,0 +1,81 @@
+# frozen_string_literal: true
+
+require 'rails_helper'
+
+describe 'statuses/show.html.haml', without_verify_partial_doubles: true do
+  before do
+    double(:api_oembed_url => '')
+    allow(view).to receive(:show_landing_strip?).and_return(true)
+    allow(view).to receive(:site_title).and_return('example site')
+    allow(view).to receive(:site_hostname).and_return('example.com')
+    allow(view).to receive(:full_asset_url).and_return('//asset.host/image.svg')
+    allow(view).to receive(:local_time)
+    allow(view).to receive(:local_time_ago)
+    allow(view).to receive(:current_account).and_return(nil)
+    assign(:instance_presenter, InstancePresenter.new)
+  end
+
+  it 'has valid author h-card and basic data for a detailed_status' do
+    alice  =  Fabricate(:account, username: 'alice', display_name: 'Alice')
+    bob    =  Fabricate(:account, username: 'bob', display_name: 'Bob')
+    status =  Fabricate(:status, account: alice, text: 'Hello World')
+    reply  =  Fabricate(:status, account: bob, thread: status, text: 'Hello Alice')
+
+    assign(:status, status)
+    assign(:account, alice)
+    assign(:descendant_threads, [])
+
+    render
+
+    mf2 = Microformats.parse(rendered)
+
+    expect(mf2.entry.url.to_s).not_to be_empty
+    expect(mf2.entry.author.name.to_s).to eq alice.display_name
+    expect(mf2.entry.author.url.to_s).not_to be_empty
+  end
+
+  it 'has valid h-cites for p-in-reply-to and p-comment' do
+    alice   =  Fabricate(:account, username: 'alice', display_name: 'Alice')
+    bob     =  Fabricate(:account, username: 'bob', display_name: 'Bob')
+    carl    =  Fabricate(:account, username: 'carl', display_name: 'Carl')
+    status  =  Fabricate(:status, account: alice, text: 'Hello World')
+    reply   =  Fabricate(:status, account: bob, thread: status, text: 'Hello Alice')
+    comment =  Fabricate(:status, account: carl, thread: reply, text: 'Hello Bob')
+
+    assign(:status, reply)
+    assign(:account, alice)
+    assign(:ancestors, reply.ancestors(1, bob))
+    assign(:descendant_threads, [{ statuses: reply.descendants(1) }])
+
+    render
+
+    mf2 = Microformats.parse(rendered)
+
+    expect(mf2.entry.url.to_s).not_to be_empty
+    expect(mf2.entry.comment.url.to_s).not_to be_empty
+    expect(mf2.entry.comment.author.name.to_s).to eq carl.display_name
+    expect(mf2.entry.comment.author.url.to_s).not_to be_empty
+
+    expect(mf2.entry.in_reply_to.url.to_s).not_to be_empty
+    expect(mf2.entry.in_reply_to.author.name.to_s).to eq alice.display_name
+    expect(mf2.entry.in_reply_to.author.url.to_s).not_to be_empty
+  end
+
+  it 'has valid opengraph tags' do
+    alice   =  Fabricate(:account, username: 'alice', display_name: 'Alice')
+    status  =  Fabricate(:status, account: alice, text: 'Hello World')
+
+    assign(:status, status)
+    assign(:account, alice)
+    assign(:descendant_threads, [])
+
+    render
+
+    header_tags = view.content_for(:header_tags)
+
+    expect(header_tags).to match(%r{<meta content=".+" property="og:title" />})
+    expect(header_tags).to match(%r{<meta content="article" property="og:type" />})
+    expect(header_tags).to match(%r{<meta content=".+" property="og:image" />})
+    expect(header_tags).to match(%r{<meta content="http://.+" property="og:url" />})
+  end
+end
diff --git a/spec/views/stream_entries/show.html.haml_spec.rb b/spec/views/stream_entries/show.html.haml_spec.rb
deleted file mode 100644
index 93f0adb99..000000000
--- a/spec/views/stream_entries/show.html.haml_spec.rb
+++ /dev/null
@@ -1,88 +0,0 @@
-# frozen_string_literal: true
-
-require 'rails_helper'
-
-describe 'stream_entries/show.html.haml', without_verify_partial_doubles: true do
-  before do
-    double(:api_oembed_url => '')
-    double(:account_stream_entry_url => '')
-    allow(view).to receive(:show_landing_strip?).and_return(true)
-    allow(view).to receive(:site_title).and_return('example site')
-    allow(view).to receive(:site_hostname).and_return('example.com')
-    allow(view).to receive(:full_asset_url).and_return('//asset.host/image.svg')
-    allow(view).to receive(:local_time)
-    allow(view).to receive(:local_time_ago)
-    allow(view).to receive(:current_account).and_return(nil)
-    assign(:instance_presenter, InstancePresenter.new)
-  end
-
-  it 'has valid author h-card and basic data for a detailed_status' do
-    alice  =  Fabricate(:account, username: 'alice', display_name: 'Alice')
-    bob    =  Fabricate(:account, username: 'bob', display_name: 'Bob')
-    status =  Fabricate(:status, account: alice, text: 'Hello World')
-    reply  =  Fabricate(:status, account: bob, thread: status, text: 'Hello Alice')
-
-    assign(:status, status)
-    assign(:stream_entry, status.stream_entry)
-    assign(:account, alice)
-    assign(:type, status.stream_entry.activity_type.downcase)
-    assign(:descendant_threads, [])
-
-    render
-
-    mf2 = Microformats.parse(rendered)
-
-    expect(mf2.entry.url.to_s).not_to be_empty
-    expect(mf2.entry.author.name.to_s).to eq alice.display_name
-    expect(mf2.entry.author.url.to_s).not_to be_empty
-  end
-
-  it 'has valid h-cites for p-in-reply-to and p-comment' do
-    alice   =  Fabricate(:account, username: 'alice', display_name: 'Alice')
-    bob     =  Fabricate(:account, username: 'bob', display_name: 'Bob')
-    carl    =  Fabricate(:account, username: 'carl', display_name: 'Carl')
-    status  =  Fabricate(:status, account: alice, text: 'Hello World')
-    reply   =  Fabricate(:status, account: bob, thread: status, text: 'Hello Alice')
-    comment =  Fabricate(:status, account: carl, thread: reply, text: 'Hello Bob')
-
-    assign(:status, reply)
-    assign(:stream_entry, reply.stream_entry)
-    assign(:account, alice)
-    assign(:type, reply.stream_entry.activity_type.downcase)
-    assign(:ancestors, reply.stream_entry.activity.ancestors(1, bob))
-    assign(:descendant_threads, [{ statuses: reply.stream_entry.activity.descendants(1) }])
-
-    render
-
-    mf2 = Microformats.parse(rendered)
-
-    expect(mf2.entry.url.to_s).not_to be_empty
-    expect(mf2.entry.comment.url.to_s).not_to be_empty
-    expect(mf2.entry.comment.author.name.to_s).to eq carl.display_name
-    expect(mf2.entry.comment.author.url.to_s).not_to be_empty
-
-    expect(mf2.entry.in_reply_to.url.to_s).not_to be_empty
-    expect(mf2.entry.in_reply_to.author.name.to_s).to eq alice.display_name
-    expect(mf2.entry.in_reply_to.author.url.to_s).not_to be_empty
-  end
-
-  it 'has valid opengraph tags' do
-    alice   =  Fabricate(:account, username: 'alice', display_name: 'Alice')
-    status  =  Fabricate(:status, account: alice, text: 'Hello World')
-
-    assign(:status, status)
-    assign(:stream_entry, status.stream_entry)
-    assign(:account, alice)
-    assign(:type, status.stream_entry.activity_type.downcase)
-    assign(:descendant_threads, [])
-
-    render
-
-    header_tags = view.content_for(:header_tags)
-
-    expect(header_tags).to match(%r{<meta content=".+" property="og:title" />})
-    expect(header_tags).to match(%r{<meta content="article" property="og:type" />})
-    expect(header_tags).to match(%r{<meta content=".+" property="og:image" />})
-    expect(header_tags).to match(%r{<meta content="http://.+" property="og:url" />})
-  end
-end
-- 
cgit 


From 63c7fe8e4892b22e80c015bf0ecb04496318623b Mon Sep 17 00:00:00 2001
From: Eugen Rochko <eugen@zeonfederated.com>
Date: Mon, 8 Jul 2019 12:03:45 +0200
Subject: Refactor controllers for statuses, accounts, and more (#11249)

---
 app/controllers/about_controller.rb                |  16 +-
 app/controllers/accounts_controller.rb             |  15 +-
 .../activitypub/collections_controller.rb          |  16 +-
 app/controllers/activitypub/inboxes_controller.rb  |   7 +-
 app/controllers/activitypub/outboxes_controller.rb |   6 +-
 app/controllers/activitypub/replies_controller.rb  |  68 +++++++++
 app/controllers/api/proofs_controller.rb           |  17 +--
 app/controllers/application_controller.rb          |   4 -
 .../concerns/account_controller_concern.rb         |  34 +----
 app/controllers/concerns/account_owned_concern.rb  |  33 +++++
 .../concerns/status_controller_concern.rb          |  87 +++++++++++
 app/controllers/custom_css_controller.rb           |   1 +
 app/controllers/emojis_controller.rb               |   5 +-
 app/controllers/follower_accounts_controller.rb    |   2 +-
 app/controllers/following_accounts_controller.rb   |   2 +-
 app/controllers/home_controller.rb                 |   2 +-
 app/controllers/intents_controller.rb              |   1 +
 app/controllers/manifests_controller.rb            |   1 +
 app/controllers/media_controller.rb                |   1 -
 app/controllers/public_timelines_controller.rb     |  14 +-
 app/controllers/remote_follow_controller.rb        |  12 +-
 app/controllers/statuses_controller.rb             | 164 ++-------------------
 app/controllers/tags_controller.rb                 |  18 ++-
 app/controllers/well_known/host_meta_controller.rb |   2 +-
 app/controllers/well_known/webfinger_controller.rb |   9 +-
 app/lib/activitypub/activity/announce.rb           |   2 +-
 app/lib/activitypub/activity/create.rb             |   2 +-
 app/lib/activitypub/activity/delete.rb             |   2 +-
 app/lib/activitypub/tag_manager.rb                 |   2 +-
 app/models/status.rb                               |   9 +-
 app/serializers/activitypub/activity_serializer.rb |   3 +
 app/serializers/activitypub/actor_serializer.rb    |   2 +
 .../activitypub/collection_serializer.rb           |   2 +
 app/serializers/activitypub/emoji_serializer.rb    |   2 +
 app/serializers/activitypub/note_serializer.rb     |   2 +
 app/services/process_hashtags_service.rb           |   2 +-
 app/views/statuses/_simple_status.html.haml        |   4 +-
 config/routes.rb                                   |   3 +-
 .../concerns/account_controller_concern_spec.rb    |   2 +-
 spec/controllers/statuses_controller_spec.rb       |   4 +-
 spec/requests/link_headers_spec.rb                 |   8 +-
 41 files changed, 299 insertions(+), 289 deletions(-)
 create mode 100644 app/controllers/activitypub/replies_controller.rb
 create mode 100644 app/controllers/concerns/account_owned_concern.rb
 create mode 100644 app/controllers/concerns/status_controller_concern.rb

(limited to 'spec')

diff --git a/app/controllers/about_controller.rb b/app/controllers/about_controller.rb
index 52a51fd62..761c7f5cd 100644
--- a/app/controllers/about_controller.rb
+++ b/app/controllers/about_controller.rb
@@ -3,11 +3,11 @@
 class AboutController < ApplicationController
   layout 'public'
 
-  before_action :set_instance_presenter, only: [:show, :more, :terms]
+  before_action :set_body_classes, only: :show
+  before_action :set_instance_presenter
+  before_action :set_expires_in
 
-  def show
-    @hide_navbar = true
-  end
+  def show; end
 
   def more; end
 
@@ -27,4 +27,12 @@ class AboutController < ApplicationController
   def set_instance_presenter
     @instance_presenter = InstancePresenter.new
   end
+
+  def set_body_classes
+    @hide_navbar = true
+  end
+
+  def set_expires_in
+    expires_in 0, public: true
+  end
 end
diff --git a/app/controllers/accounts_controller.rb b/app/controllers/accounts_controller.rb
index 065707378..3184a73cb 100644
--- a/app/controllers/accounts_controller.rb
+++ b/app/controllers/accounts_controller.rb
@@ -6,13 +6,13 @@ class AccountsController < ApplicationController
   include AccountControllerConcern
 
   before_action :set_cache_headers
+  before_action :set_body_classes
 
   def show
     respond_to do |format|
       format.html do
-        mark_cacheable! unless user_signed_in?
+        expires_in 0, public: true unless user_signed_in?
 
-        @body_classes      = 'with-modals'
         @pinned_statuses   = []
         @endorsed_accounts = @account.endorsed_accounts.to_a.sample(4)
 
@@ -32,22 +32,25 @@ class AccountsController < ApplicationController
       end
 
       format.rss do
-        mark_cacheable!
+        expires_in 0, public: true
 
         @statuses = cache_collection(default_statuses.without_reblogs.without_replies.limit(PAGE_SIZE), Status)
         render xml: RSS::AccountSerializer.render(@account, @statuses)
       end
 
       format.json do
-        render_cached_json(['activitypub', 'actor', @account], content_type: 'application/activity+json') do
-          ActiveModelSerializers::SerializableResource.new(@account, serializer: ActivityPub::ActorSerializer, adapter: ActivityPub::Adapter)
-        end
+        expires_in 3.minutes, public: true
+        render json: @account, content_type: 'application/activity+json', serializer: ActivityPub::ActorSerializer, adapter: ActivityPub::Adapter
       end
     end
   end
 
   private
 
+  def set_body_classes
+    @body_classes = 'with-modals'
+  end
+
   def show_pinned_statuses?
     [replies_requested?, media_requested?, tag_requested?, params[:max_id].present?, params[:min_id].present?].none?
   end
diff --git a/app/controllers/activitypub/collections_controller.rb b/app/controllers/activitypub/collections_controller.rb
index 012c3c538..dd2f111b0 100644
--- a/app/controllers/activitypub/collections_controller.rb
+++ b/app/controllers/activitypub/collections_controller.rb
@@ -2,29 +2,19 @@
 
 class ActivityPub::CollectionsController < Api::BaseController
   include SignatureVerification
+  include AccountOwnedConcern
 
-  before_action :set_account
   before_action :set_size
   before_action :set_statuses
   before_action :set_cache_headers
 
   def show
-    render_cached_json(['activitypub', 'collection', @account, params[:id]], content_type: 'application/activity+json') do
-      ActiveModelSerializers::SerializableResource.new(
-        collection_presenter,
-        serializer: ActivityPub::CollectionSerializer,
-        adapter: ActivityPub::Adapter,
-        skip_activities: true
-      )
-    end
+    expires_in 3.minutes, public: true
+    render json: collection_presenter, content_type: 'application/activity+json', serializer: ActivityPub::CollectionSerializer, adapter: ActivityPub::Adapter, skip_activities: true
   end
 
   private
 
-  def set_account
-    @account = Account.find_local!(params[:account_username])
-  end
-
   def set_statuses
     @statuses = scope_for_collection
     @statuses = cache_collection(@statuses, Status)
diff --git a/app/controllers/activitypub/inboxes_controller.rb b/app/controllers/activitypub/inboxes_controller.rb
index e2cd8eaed..9be0676e1 100644
--- a/app/controllers/activitypub/inboxes_controller.rb
+++ b/app/controllers/activitypub/inboxes_controller.rb
@@ -3,8 +3,7 @@
 class ActivityPub::InboxesController < Api::BaseController
   include SignatureVerification
   include JsonLdHelper
-
-  before_action :set_account
+  include AccountOwnedConcern
 
   def create
     if unknown_deleted_account?
@@ -27,8 +26,8 @@ class ActivityPub::InboxesController < Api::BaseController
     false
   end
 
-  def set_account
-    @account = Account.find_local!(params[:account_username]) if params[:account_username]
+  def account_required?
+    params[:account_username].present?
   end
 
   def body
diff --git a/app/controllers/activitypub/outboxes_controller.rb b/app/controllers/activitypub/outboxes_controller.rb
index 5147afbf7..4c0b769f0 100644
--- a/app/controllers/activitypub/outboxes_controller.rb
+++ b/app/controllers/activitypub/outboxes_controller.rb
@@ -4,8 +4,8 @@ class ActivityPub::OutboxesController < Api::BaseController
   LIMIT = 20
 
   include SignatureVerification
+  include AccountOwnedConcern
 
-  before_action :set_account
   before_action :set_statuses
   before_action :set_cache_headers
 
@@ -17,10 +17,6 @@ class ActivityPub::OutboxesController < Api::BaseController
 
   private
 
-  def set_account
-    @account = Account.find_local!(params[:account_username])
-  end
-
   def outbox_presenter
     if page_requested?
       ActivityPub::CollectionPresenter.new(
diff --git a/app/controllers/activitypub/replies_controller.rb b/app/controllers/activitypub/replies_controller.rb
new file mode 100644
index 000000000..99b7b310f
--- /dev/null
+++ b/app/controllers/activitypub/replies_controller.rb
@@ -0,0 +1,68 @@
+# frozen_string_literal: true
+
+class ActivityPub::RepliesController < Api::BaseController
+  include SignatureAuthentication
+  include Authorization
+  include AccountOwnedConcern
+
+  DESCENDANTS_LIMIT = 60
+
+  before_action :set_status
+  before_action :set_cache_headers
+  before_action :set_replies
+
+  def index
+    render json: replies_collection_presenter, serializer: ActivityPub::CollectionSerializer, adapter: ActivityPub::Adapter, content_type: 'application/activity+json', skip_activities: true
+  end
+
+  private
+
+  def set_status
+    @status = @account.statuses.find(params[:status_id])
+    authorize @status, :show?
+  rescue Mastodon::NotPermittedError
+    raise ActiveRecord::RecordNotFound
+  end
+
+  def set_replies
+    @replies = page_params[:other_accounts] ? Status.where.not(account_id: @account.id) : @account.statuses
+    @replies = @replies.where(in_reply_to_id: @status.id, visibility: [:public, :unlisted])
+    @replies = @replies.paginate_by_min_id(DESCENDANTS_LIMIT, params[:min_id])
+  end
+
+  def replies_collection_presenter
+    page = ActivityPub::CollectionPresenter.new(
+      id: account_status_replies_url(@account, @status, page_params),
+      type: :unordered,
+      part_of: account_status_replies_url(@account, @status),
+      next: next_page,
+      items: @replies.map { |status| status.local ? status : status.id }
+    )
+
+    return page if page_requested?
+
+    ActivityPub::CollectionPresenter.new(
+      id: account_status_replies_url(@account, @status),
+      type: :unordered,
+      first: page
+    )
+  end
+
+  def page_requested?
+    params[:page] == 'true'
+  end
+
+  def next_page
+    account_status_replies_url(
+      @account,
+      @status,
+      page: true,
+      min_id: @replies&.last&.id,
+      other_accounts: !(@replies&.last&.account_id == @account.id && @replies.size == DESCENDANTS_LIMIT)
+    )
+  end
+
+  def page_params
+    params_slice(:other_accounts, :min_id).merge(page: true)
+  end
+end
diff --git a/app/controllers/api/proofs_controller.rb b/app/controllers/api/proofs_controller.rb
index a84ad2014..a98599eee 100644
--- a/app/controllers/api/proofs_controller.rb
+++ b/app/controllers/api/proofs_controller.rb
@@ -1,10 +1,9 @@
 # frozen_string_literal: true
 
 class Api::ProofsController < Api::BaseController
-  before_action :set_account
+  include AccountOwnedConcern
+
   before_action :set_provider
-  before_action :check_account_approval
-  before_action :check_account_suspension
 
   def index
     render json: @account, serializer: @provider.serializer_class
@@ -16,15 +15,7 @@ class Api::ProofsController < Api::BaseController
     @provider = ProofProvider.find(params[:provider]) || raise(ActiveRecord::RecordNotFound)
   end
 
-  def set_account
-    @account = Account.find_local!(params[:username])
-  end
-
-  def check_account_approval
-    not_found if @account.user_pending?
-  end
-
-  def check_account_suspension
-    gone if @account.suspended?
+  def username_param
+    params[:username]
   end
 end
diff --git a/app/controllers/application_controller.rb b/app/controllers/application_controller.rb
index bd8000db0..cc8b8e4da 100644
--- a/app/controllers/application_controller.rb
+++ b/app/controllers/application_controller.rb
@@ -154,8 +154,4 @@ class ApplicationController < ActionController::Base
   def set_cache_headers
     response.headers['Vary'] = 'Accept'
   end
-
-  def mark_cacheable!
-    expires_in 0, public: true
-  end
 end
diff --git a/app/controllers/concerns/account_controller_concern.rb b/app/controllers/concerns/account_controller_concern.rb
index 1c422096c..287a930da 100644
--- a/app/controllers/concerns/account_controller_concern.rb
+++ b/app/controllers/concerns/account_controller_concern.rb
@@ -3,24 +3,19 @@
 module AccountControllerConcern
   extend ActiveSupport::Concern
 
+  include AccountOwnedConcern
+
   FOLLOW_PER_PAGE = 12
 
   included do
     layout 'public'
 
-    before_action :set_account
-    before_action :check_account_approval
-    before_action :check_account_suspension
     before_action :set_instance_presenter
     before_action :set_link_headers
   end
 
   private
 
-  def set_account
-    @account = Account.find_local!(username_param)
-  end
-
   def set_instance_presenter
     @instance_presenter = InstancePresenter.new
   end
@@ -29,27 +24,15 @@ module AccountControllerConcern
     response.headers['Link'] = LinkHeader.new(
       [
         webfinger_account_link,
-        atom_account_url_link,
         actor_url_link,
       ]
     )
   end
 
-  def username_param
-    params[:account_username]
-  end
-
   def webfinger_account_link
     [
       webfinger_account_url,
-      [%w(rel lrdd), %w(type application/xrd+xml)],
-    ]
-  end
-
-  def atom_account_url_link
-    [
-      account_url(@account, format: 'atom'),
-      [%w(rel alternate), %w(type application/atom+xml)],
+      [%w(rel lrdd), %w(type application/jrd+json)],
     ]
   end
 
@@ -63,15 +46,4 @@ module AccountControllerConcern
   def webfinger_account_url
     webfinger_url(resource: @account.to_webfinger_s)
   end
-
-  def check_account_approval
-    not_found if @account.user_pending?
-  end
-
-  def check_account_suspension
-    if @account.suspended?
-      expires_in(3.minutes, public: true)
-      gone
-    end
-  end
 end
diff --git a/app/controllers/concerns/account_owned_concern.rb b/app/controllers/concerns/account_owned_concern.rb
new file mode 100644
index 000000000..99c240fe9
--- /dev/null
+++ b/app/controllers/concerns/account_owned_concern.rb
@@ -0,0 +1,33 @@
+# frozen_string_literal: true
+
+module AccountOwnedConcern
+  extend ActiveSupport::Concern
+
+  included do
+    before_action :set_account, if: :account_required?
+    before_action :check_account_approval, if: :account_required?
+    before_action :check_account_suspension, if: :account_required?
+  end
+
+  private
+
+  def account_required?
+    true
+  end
+
+  def set_account
+    @account = Account.find_local!(username_param)
+  end
+
+  def username_param
+    params[:account_username]
+  end
+
+  def check_account_approval
+    not_found if @account.local? && @account.user_pending?
+  end
+
+  def check_account_suspension
+    expires_in(3.minutes, public: true) && gone if @account.suspended?
+  end
+end
diff --git a/app/controllers/concerns/status_controller_concern.rb b/app/controllers/concerns/status_controller_concern.rb
new file mode 100644
index 000000000..62a7cf508
--- /dev/null
+++ b/app/controllers/concerns/status_controller_concern.rb
@@ -0,0 +1,87 @@
+# frozen_string_literal: true
+
+module StatusControllerConcern
+  extend ActiveSupport::Concern
+
+  ANCESTORS_LIMIT         = 40
+  DESCENDANTS_LIMIT       = 60
+  DESCENDANTS_DEPTH_LIMIT = 20
+
+  def create_descendant_thread(starting_depth, statuses)
+    depth = starting_depth + statuses.size
+
+    if depth < DESCENDANTS_DEPTH_LIMIT
+      {
+        statuses: statuses,
+        starting_depth: starting_depth,
+      }
+    else
+      next_status = statuses.pop
+
+      {
+        statuses: statuses,
+        starting_depth: starting_depth,
+        next_status: next_status,
+      }
+    end
+  end
+
+  def set_ancestors
+    @ancestors     = @status.reply? ? cache_collection(@status.ancestors(ANCESTORS_LIMIT, current_account), Status) : []
+    @next_ancestor = @ancestors.size < ANCESTORS_LIMIT ? nil : @ancestors.shift
+  end
+
+  def set_descendants
+    @max_descendant_thread_id   = params[:max_descendant_thread_id]&.to_i
+    @since_descendant_thread_id = params[:since_descendant_thread_id]&.to_i
+
+    descendants = cache_collection(
+      @status.descendants(
+        DESCENDANTS_LIMIT,
+        current_account,
+        @max_descendant_thread_id,
+        @since_descendant_thread_id,
+        DESCENDANTS_DEPTH_LIMIT
+      ),
+      Status
+    )
+
+    @descendant_threads = []
+
+    if descendants.present?
+      statuses       = [descendants.first]
+      starting_depth = 0
+
+      descendants.drop(1).each_with_index do |descendant, index|
+        if descendants[index].id == descendant.in_reply_to_id
+          statuses << descendant
+        else
+          @descendant_threads << create_descendant_thread(starting_depth, statuses)
+
+          # The thread is broken, assume it's a reply to the root status
+          starting_depth = 0
+
+          # ... unless we can find its ancestor in one of the already-processed threads
+          @descendant_threads.reverse_each do |descendant_thread|
+            statuses = descendant_thread[:statuses]
+
+            index = statuses.find_index do |thread_status|
+              thread_status.id == descendant.in_reply_to_id
+            end
+
+            if index.present?
+              starting_depth = descendant_thread[:starting_depth] + index + 1
+              break
+            end
+          end
+
+          statuses = [descendant]
+        end
+      end
+
+      @descendant_threads << create_descendant_thread(starting_depth, statuses)
+    end
+
+    @max_descendant_thread_id = @descendant_threads.pop[:statuses].first.id if descendants.size >= DESCENDANTS_LIMIT
+  end
+end
diff --git a/app/controllers/custom_css_controller.rb b/app/controllers/custom_css_controller.rb
index 6e80feaf8..7f4dcfcfe 100644
--- a/app/controllers/custom_css_controller.rb
+++ b/app/controllers/custom_css_controller.rb
@@ -6,6 +6,7 @@ class CustomCssController < ApplicationController
   before_action :set_cache_headers
 
   def show
+    expires 3.minutes, public: true
     render plain: Setting.custom_css || '', content_type: 'text/css'
   end
 end
diff --git a/app/controllers/emojis_controller.rb b/app/controllers/emojis_controller.rb
index 3feb08132..fe4c19cad 100644
--- a/app/controllers/emojis_controller.rb
+++ b/app/controllers/emojis_controller.rb
@@ -7,9 +7,8 @@ class EmojisController < ApplicationController
   def show
     respond_to do |format|
       format.json do
-        render_cached_json(['activitypub', 'emoji', @emoji], content_type: 'application/activity+json') do
-          ActiveModelSerializers::SerializableResource.new(@emoji, serializer: ActivityPub::EmojiSerializer, adapter: ActivityPub::Adapter)
-        end
+        expires_in 3.minutes, public: true
+        render json: @emoji, content_type: 'application/activity+json', serializer: ActivityPub::EmojiSerializer, adapter: ActivityPub::Adapter
       end
     end
   end
diff --git a/app/controllers/follower_accounts_controller.rb b/app/controllers/follower_accounts_controller.rb
index 415abe10c..8baa64490 100644
--- a/app/controllers/follower_accounts_controller.rb
+++ b/app/controllers/follower_accounts_controller.rb
@@ -8,7 +8,7 @@ class FollowerAccountsController < ApplicationController
   def index
     respond_to do |format|
       format.html do
-        mark_cacheable! unless user_signed_in?
+        expires_in 0, public: true unless user_signed_in?
 
         next if @account.user_hides_network?
 
diff --git a/app/controllers/following_accounts_controller.rb b/app/controllers/following_accounts_controller.rb
index 948725664..4d1ea4594 100644
--- a/app/controllers/following_accounts_controller.rb
+++ b/app/controllers/following_accounts_controller.rb
@@ -8,7 +8,7 @@ class FollowingAccountsController < ApplicationController
   def index
     respond_to do |format|
       format.html do
-        mark_cacheable! unless user_signed_in?
+        expires_in 0, public: true unless user_signed_in?
 
         next if @account.user_hides_network?
 
diff --git a/app/controllers/home_controller.rb b/app/controllers/home_controller.rb
index 85622a7b5..d1c525134 100644
--- a/app/controllers/home_controller.rb
+++ b/app/controllers/home_controller.rb
@@ -21,7 +21,7 @@ class HomeController < ApplicationController
       when 'statuses'
         status = Status.find_by(id: matches[2])
 
-        if status && (status.public_visibility? || status.unlisted_visibility?)
+        if status&.distributable?
           redirect_to(ActivityPub::TagManager.instance.url_for(status))
           return
         end
diff --git a/app/controllers/intents_controller.rb b/app/controllers/intents_controller.rb
index 9f41cf48a..ca89fc7fe 100644
--- a/app/controllers/intents_controller.rb
+++ b/app/controllers/intents_controller.rb
@@ -2,6 +2,7 @@
 
 class IntentsController < ApplicationController
   before_action :check_uri
+
   rescue_from Addressable::URI::InvalidURIError, with: :handle_invalid_uri
 
   def show
diff --git a/app/controllers/manifests_controller.rb b/app/controllers/manifests_controller.rb
index 332d845d8..1e5db4393 100644
--- a/app/controllers/manifests_controller.rb
+++ b/app/controllers/manifests_controller.rb
@@ -4,6 +4,7 @@ class ManifestsController < ApplicationController
   skip_before_action :store_current_location
 
   def show
+    expires_in 3.minutes, public: true
     render json: InstancePresenter.new, serializer: ManifestSerializer
   end
 end
diff --git a/app/controllers/media_controller.rb b/app/controllers/media_controller.rb
index d44b52d26..b3b7519a1 100644
--- a/app/controllers/media_controller.rb
+++ b/app/controllers/media_controller.rb
@@ -31,7 +31,6 @@ class MediaController < ApplicationController
   def verify_permitted_status!
     authorize @media_attachment.status, :show?
   rescue Mastodon::NotPermittedError
-    # Reraise in order to get a 404 instead of a 403 error code
     raise ActiveRecord::RecordNotFound
   end
 
diff --git a/app/controllers/public_timelines_controller.rb b/app/controllers/public_timelines_controller.rb
index 53d4472d8..23506b990 100644
--- a/app/controllers/public_timelines_controller.rb
+++ b/app/controllers/public_timelines_controller.rb
@@ -8,20 +8,16 @@ class PublicTimelinesController < ApplicationController
   before_action :set_instance_presenter
 
   def show
-    respond_to do |format|
-      format.html do
-        @initial_state_json = ActiveModelSerializers::SerializableResource.new(
-          InitialStatePresenter.new(settings: { known_fediverse: Setting.show_known_fediverse_at_about_page }, token: current_session&.token),
-          serializer: InitialStateSerializer
-        ).to_json
-      end
-    end
+    @initial_state_json = ActiveModelSerializers::SerializableResource.new(
+      InitialStatePresenter.new(settings: { known_fediverse: Setting.show_known_fediverse_at_about_page }, token: current_session&.token),
+      serializer: InitialStateSerializer
+    ).to_json
   end
 
   private
 
   def check_enabled
-    raise ActiveRecord::RecordNotFound unless Setting.timeline_preview
+    not_found unless Setting.timeline_preview
   end
 
   def set_body_classes
diff --git a/app/controllers/remote_follow_controller.rb b/app/controllers/remote_follow_controller.rb
index 8ba331cd1..0fb71d335 100644
--- a/app/controllers/remote_follow_controller.rb
+++ b/app/controllers/remote_follow_controller.rb
@@ -1,10 +1,10 @@
 # frozen_string_literal: true
 
 class RemoteFollowController < ApplicationController
+  include AccountOwnedConcern
+
   layout 'modal'
 
-  before_action :set_account
-  before_action :gone, if: :suspended_account?
   before_action :set_body_classes
 
   def new
@@ -32,14 +32,6 @@ class RemoteFollowController < ApplicationController
     { acct: session[:remote_follow] }
   end
 
-  def set_account
-    @account = Account.find_local!(params[:account_username])
-  end
-
-  def suspended_account?
-    @account.suspended?
-  end
-
   def set_body_classes
     @body_classes = 'modal-layout'
     @hide_header  = true
diff --git a/app/controllers/statuses_controller.rb b/app/controllers/statuses_controller.rb
index 776099ca8..13ce5c691 100644
--- a/app/controllers/statuses_controller.rb
+++ b/app/controllers/statuses_controller.rb
@@ -1,24 +1,21 @@
 # frozen_string_literal: true
 
 class StatusesController < ApplicationController
+  include StatusControllerConcern
   include SignatureAuthentication
   include Authorization
-
-  ANCESTORS_LIMIT         = 40
-  DESCENDANTS_LIMIT       = 60
-  DESCENDANTS_DEPTH_LIMIT = 20
+  include AccountOwnedConcern
 
   layout 'public'
 
-  before_action :set_account
   before_action :set_status
   before_action :set_instance_presenter
   before_action :set_link_headers
-  before_action :check_account_suspension
   before_action :redirect_to_original, only: [:show]
   before_action :set_referrer_policy_header, only: [:show]
   before_action :set_cache_headers
-  before_action :set_replies, only: [:replies]
+  before_action :set_body_classes
+  before_action :set_autoplay, only: :embed
 
   content_security_policy only: :embed do |p|
     p.frame_ancestors(false)
@@ -28,25 +25,20 @@ class StatusesController < ApplicationController
     respond_to do |format|
       format.html do
         expires_in 10.seconds, public: true if current_account.nil?
-
-        @body_classes = 'with-modals'
-
         set_ancestors
         set_descendants
       end
 
       format.json do
-        render_cached_json(['activitypub', 'note', @status], content_type: 'application/activity+json', public: @status.distributable?) do
-          ActiveModelSerializers::SerializableResource.new(@status, serializer: ActivityPub::NoteSerializer, adapter: ActivityPub::Adapter)
-        end
+        expires_in 3.minutes, public: @status.distributable?
+        render json: @status, content_type: 'application/activity+json', serializer: ActivityPub::NoteSerializer, adapter: ActivityPub::Adapter
       end
     end
   end
 
   def activity
-    render_cached_json(['activitypub', 'activity', @status], content_type: 'application/activity+json', public: @status.distributable?) do
-      ActiveModelSerializers::SerializableResource.new(@status, serializer: ActivityPub::ActivitySerializer, adapter: ActivityPub::Adapter)
-    end
+    expires_in 3.minutes, public: @status.distributable?
+    render json: @status, content_type: 'application/activity+json', serializer: ActivityPub::ActivitySerializer, adapter: ActivityPub::Adapter
   end
 
   def embed
@@ -54,120 +46,14 @@ class StatusesController < ApplicationController
 
     expires_in 180, public: true
     response.headers['X-Frame-Options'] = 'ALLOWALL'
-    @autoplay = ActiveModel::Type::Boolean.new.cast(params[:autoplay])
 
     render layout: 'embedded'
   end
 
-  def replies
-    render json: replies_collection_presenter,
-           serializer: ActivityPub::CollectionSerializer,
-           adapter: ActivityPub::Adapter,
-           content_type: 'application/activity+json',
-           skip_activities: true
-  end
-
   private
 
-  def replies_collection_presenter
-    page = ActivityPub::CollectionPresenter.new(
-      id: replies_account_status_url(@account, @status, page_params),
-      type: :unordered,
-      part_of: replies_account_status_url(@account, @status),
-      next: next_page,
-      items: @replies.map { |status| status.local ? status : status.id }
-    )
-    if page_requested?
-      page
-    else
-      ActivityPub::CollectionPresenter.new(
-        id: replies_account_status_url(@account, @status),
-        type: :unordered,
-        first: page
-      )
-    end
-  end
-
-  def create_descendant_thread(starting_depth, statuses)
-    depth = starting_depth + statuses.size
-
-    if depth < DESCENDANTS_DEPTH_LIMIT
-      {
-        statuses: statuses,
-        starting_depth: starting_depth,
-      }
-    else
-      next_status = statuses.pop
-
-      {
-        statuses: statuses,
-        starting_depth: starting_depth,
-        next_status: next_status,
-      }
-    end
-  end
-
-  def set_account
-    @account = Account.find_local!(params[:account_username])
-  end
-
-  def set_ancestors
-    @ancestors     = @status.reply? ? cache_collection(@status.ancestors(ANCESTORS_LIMIT, current_account), Status) : []
-    @next_ancestor = @ancestors.size < ANCESTORS_LIMIT ? nil : @ancestors.shift
-  end
-
-  def set_descendants
-    @max_descendant_thread_id   = params[:max_descendant_thread_id]&.to_i
-    @since_descendant_thread_id = params[:since_descendant_thread_id]&.to_i
-
-    descendants = cache_collection(
-      @status.descendants(
-        DESCENDANTS_LIMIT,
-        current_account,
-        @max_descendant_thread_id,
-        @since_descendant_thread_id,
-        DESCENDANTS_DEPTH_LIMIT
-      ),
-      Status
-    )
-
-    @descendant_threads = []
-
-    if descendants.present?
-      statuses       = [descendants.first]
-      starting_depth = 0
-
-      descendants.drop(1).each_with_index do |descendant, index|
-        if descendants[index].id == descendant.in_reply_to_id
-          statuses << descendant
-        else
-          @descendant_threads << create_descendant_thread(starting_depth, statuses)
-
-          # The thread is broken, assume it's a reply to the root status
-          starting_depth = 0
-
-          # ... unless we can find its ancestor in one of the already-processed threads
-          @descendant_threads.reverse_each do |descendant_thread|
-            statuses = descendant_thread[:statuses]
-
-            index = statuses.find_index do |thread_status|
-              thread_status.id == descendant.in_reply_to_id
-            end
-
-            if index.present?
-              starting_depth = descendant_thread[:starting_depth] + index + 1
-              break
-            end
-          end
-
-          statuses = [descendant]
-        end
-      end
-
-      @descendant_threads << create_descendant_thread(starting_depth, statuses)
-    end
-
-    @max_descendant_thread_id = @descendant_threads.pop[:statuses].first.id if descendants.size >= DESCENDANTS_LIMIT
+  def set_body_classes
+    @body_classes = 'with-modals'
   end
 
   def set_link_headers
@@ -185,39 +71,15 @@ class StatusesController < ApplicationController
     @instance_presenter = InstancePresenter.new
   end
 
-  def check_account_suspension
-    gone if @account.suspended?
-  end
-
   def redirect_to_original
     redirect_to ActivityPub::TagManager.instance.url_for(@status.reblog) if @status.reblog?
   end
 
   def set_referrer_policy_header
-    return if @status.public_visibility? || @status.unlisted_visibility?
-    response.headers['Referrer-Policy'] = 'origin'
-  end
-
-  def page_requested?
-    params[:page] == 'true'
-  end
-
-  def set_replies
-    @replies = page_params[:other_accounts] ? Status.where.not(account_id: @account.id) : @account.statuses
-    @replies = @replies.where(in_reply_to_id: @status.id, visibility: [:public, :unlisted])
-    @replies = @replies.paginate_by_min_id(DESCENDANTS_LIMIT, params[:min_id])
-  end
-
-  def next_page
-    last_reply = @replies.last
-    return if last_reply.nil?
-    same_account = last_reply.account_id == @account.id
-    return unless same_account || @replies.size == DESCENDANTS_LIMIT
-    same_account = false unless @replies.size == DESCENDANTS_LIMIT
-    replies_account_status_url(@account, @status, page: true, min_id: last_reply.id, other_accounts: !same_account)
+    response.headers['Referrer-Policy'] = 'origin' unless @status.distributable?
   end
 
-  def page_params
-    { page: true, other_accounts: params[:other_accounts], min_id: params[:min_id] }.compact
+  def set_autoplay
+    @autoplay = truthy_param?(:autoplay)
   end
 end
diff --git a/app/controllers/tags_controller.rb b/app/controllers/tags_controller.rb
index 66b184901..2ecce0ca2 100644
--- a/app/controllers/tags_controller.rb
+++ b/app/controllers/tags_controller.rb
@@ -5,14 +5,15 @@ class TagsController < ApplicationController
 
   layout 'public'
 
+  before_action :set_tag
   before_action :set_body_classes
   before_action :set_instance_presenter
 
   def show
-    @tag = Tag.find_normalized!(params[:id])
-
     respond_to do |format|
       format.html do
+        expires_in 0, public: true
+
         @initial_state_json = ActiveModelSerializers::SerializableResource.new(
           InitialStatePresenter.new(settings: {}, token: current_session&.token),
           serializer: InitialStateSerializer
@@ -20,6 +21,8 @@ class TagsController < ApplicationController
       end
 
       format.rss do
+        expires_in 0, public: true
+
         @statuses = HashtagQueryService.new.call(@tag, params.slice(:any, :all, :none)).limit(PAGE_SIZE)
         @statuses = cache_collection(@statuses, Status)
 
@@ -27,19 +30,22 @@ class TagsController < ApplicationController
       end
 
       format.json do
+        expires_in 3.minutes, public: true
+
         @statuses = HashtagQueryService.new.call(@tag, params.slice(:any, :all, :none), current_account, params[:local]).paginate_by_max_id(PAGE_SIZE, params[:max_id])
         @statuses = cache_collection(@statuses, Status)
 
-        render json: collection_presenter,
-               serializer: ActivityPub::CollectionSerializer,
-               adapter: ActivityPub::Adapter,
-               content_type: 'application/activity+json'
+        render json: collection_presenter, serializer: ActivityPub::CollectionSerializer, adapter: ActivityPub::Adapter, content_type: 'application/activity+json'
       end
     end
   end
 
   private
 
+  def set_tag
+    @tag = Tag.find_normalized!(params[:id])
+  end
+
   def set_body_classes
     @body_classes = 'with-modals'
   end
diff --git a/app/controllers/well_known/host_meta_controller.rb b/app/controllers/well_known/host_meta_controller.rb
index 5fb70288a..2e9298c4a 100644
--- a/app/controllers/well_known/host_meta_controller.rb
+++ b/app/controllers/well_known/host_meta_controller.rb
@@ -13,7 +13,7 @@ module WellKnown
         format.xml { render content_type: 'application/xrd+xml' }
       end
 
-      expires_in(3.days, public: true)
+      expires_in 3.days, public: true
     end
   end
 end
diff --git a/app/controllers/well_known/webfinger_controller.rb b/app/controllers/well_known/webfinger_controller.rb
index 28654b61d..53f7f1e27 100644
--- a/app/controllers/well_known/webfinger_controller.rb
+++ b/app/controllers/well_known/webfinger_controller.rb
@@ -19,7 +19,7 @@ module WellKnown
         end
       end
 
-      expires_in(3.days, public: true)
+      expires_in 3.days, public: true
     rescue ActiveRecord::RecordNotFound
       head 404
     end
@@ -27,12 +27,9 @@ module WellKnown
     private
 
     def username_from_resource
-      resource_user = resource_param
-
+      resource_user    = resource_param
       username, domain = resource_user.split('@')
-      if Rails.configuration.x.alternate_domains.include?(domain)
-        resource_user = "#{username}@#{Rails.configuration.x.local_domain}"
-      end
+      resource_user    = "#{username}@#{Rails.configuration.x.local_domain}" if Rails.configuration.x.alternate_domains.include?(domain)
 
       WebfingerResource.new(resource_user).username
     end
diff --git a/app/lib/activitypub/activity/announce.rb b/app/lib/activitypub/activity/announce.rb
index 1aa6ee9ec..34c646668 100644
--- a/app/lib/activitypub/activity/announce.rb
+++ b/app/lib/activitypub/activity/announce.rb
@@ -40,7 +40,7 @@ class ActivityPub::Activity::Announce < ActivityPub::Activity
   end
 
   def announceable?(status)
-    status.account_id == @account.id || status.public_visibility? || status.unlisted_visibility?
+    status.account_id == @account.id || status.distributable?
   end
 
   def related_to_local_activity?
diff --git a/app/lib/activitypub/activity/create.rb b/app/lib/activitypub/activity/create.rb
index 00f0dd42d..5849c20d7 100644
--- a/app/lib/activitypub/activity/create.rb
+++ b/app/lib/activitypub/activity/create.rb
@@ -42,7 +42,7 @@ class ActivityPub::Activity::Create < ActivityPub::Activity
     resolve_thread(@status)
     fetch_replies(@status)
     distribute(@status)
-    forward_for_reply if @status.public_visibility? || @status.unlisted_visibility?
+    forward_for_reply if @status.distributable?
   end
 
   def find_existing_status
diff --git a/app/lib/activitypub/activity/delete.rb b/app/lib/activitypub/activity/delete.rb
index 0eb14b89c..1f2b40c15 100644
--- a/app/lib/activitypub/activity/delete.rb
+++ b/app/lib/activitypub/activity/delete.rb
@@ -31,7 +31,7 @@ class ActivityPub::Activity::Delete < ActivityPub::Activity
 
     return if @status.nil?
 
-    if @status.public_visibility? || @status.unlisted_visibility?
+    if @status.distributable?
       forward_for_reply
       forward_for_reblogs
     end
diff --git a/app/lib/activitypub/tag_manager.rb b/app/lib/activitypub/tag_manager.rb
index 595291342..4d452f290 100644
--- a/app/lib/activitypub/tag_manager.rb
+++ b/app/lib/activitypub/tag_manager.rb
@@ -51,7 +51,7 @@ class ActivityPub::TagManager
   def replies_uri_for(target, page_params = nil)
     raise ArgumentError, 'target must be a local activity' unless %i(note comment activity).include?(target.object_type) && target.local?
 
-    replies_account_status_url(target.account, target, page_params)
+    account_status_replies_url(target.account, target, page_params)
   end
 
   # Primary audience of a status
diff --git a/app/models/status.rb b/app/models/status.rb
index 906756e85..6f1e35e4a 100644
--- a/app/models/status.rb
+++ b/app/models/status.rb
@@ -193,7 +193,7 @@ class Status < ApplicationRecord
   end
 
   def hidden?
-    private_visibility? || direct_visibility? || limited_visibility?
+    !distributable?
   end
 
   def distributable?
@@ -446,7 +446,8 @@ class Status < ApplicationRecord
   end
 
   def update_statistics
-    return unless public_visibility? || unlisted_visibility?
+    return unless distributable?
+
     ActivityTracker.increment('activity:statuses:local')
   end
 
@@ -455,7 +456,7 @@ class Status < ApplicationRecord
 
     account&.increment_count!(:statuses_count)
     reblog&.increment_count!(:reblogs_count) if reblog?
-    thread&.increment_count!(:replies_count) if in_reply_to_id.present? && (public_visibility? || unlisted_visibility?)
+    thread&.increment_count!(:replies_count) if in_reply_to_id.present? && distributable?
   end
 
   def decrement_counter_caches
@@ -463,7 +464,7 @@ class Status < ApplicationRecord
 
     account&.decrement_count!(:statuses_count)
     reblog&.decrement_count!(:reblogs_count) if reblog?
-    thread&.decrement_count!(:replies_count) if in_reply_to_id.present? && (public_visibility? || unlisted_visibility?)
+    thread&.decrement_count!(:replies_count) if in_reply_to_id.present? && distributable?
   end
 
   def unlink_from_conversations
diff --git a/app/serializers/activitypub/activity_serializer.rb b/app/serializers/activitypub/activity_serializer.rb
index c06d5c87c..fdedbc9d1 100644
--- a/app/serializers/activitypub/activity_serializer.rb
+++ b/app/serializers/activitypub/activity_serializer.rb
@@ -1,9 +1,12 @@
 # frozen_string_literal: true
 
 class ActivityPub::ActivitySerializer < ActivityPub::Serializer
+  cache key: 'activity', expires_in: 3.minutes
+
   attributes :id, :type, :actor, :published, :to, :cc
 
   has_one :proper, key: :object, serializer: ActivityPub::NoteSerializer, if: :serialize_object?
+
   attribute :proper_uri, key: :object, unless: :serialize_object?
   attribute :atom_uri, if: :announce?
 
diff --git a/app/serializers/activitypub/actor_serializer.rb b/app/serializers/activitypub/actor_serializer.rb
index 0644219fb..ab7be27f6 100644
--- a/app/serializers/activitypub/actor_serializer.rb
+++ b/app/serializers/activitypub/actor_serializer.rb
@@ -3,6 +3,8 @@
 class ActivityPub::ActorSerializer < ActivityPub::Serializer
   include RoutingHelper
 
+  cache key: 'actor', expires_in: 3.minutes
+
   context :security
 
   context_extensions :manually_approves_followers, :featured, :also_known_as,
diff --git a/app/serializers/activitypub/collection_serializer.rb b/app/serializers/activitypub/collection_serializer.rb
index da1ba735f..9dd8134d3 100644
--- a/app/serializers/activitypub/collection_serializer.rb
+++ b/app/serializers/activitypub/collection_serializer.rb
@@ -7,6 +7,8 @@ class ActivityPub::CollectionSerializer < ActivityPub::Serializer
     super
   end
 
+  cache key: 'collection', expires_in: 3.minutes
+
   attribute :id, if: -> { object.id.present? }
   attribute :type
   attribute :total_items, if: -> { object.size.present? }
diff --git a/app/serializers/activitypub/emoji_serializer.rb b/app/serializers/activitypub/emoji_serializer.rb
index 4dc38f3ea..08df25d7d 100644
--- a/app/serializers/activitypub/emoji_serializer.rb
+++ b/app/serializers/activitypub/emoji_serializer.rb
@@ -3,6 +3,8 @@
 class ActivityPub::EmojiSerializer < ActivityPub::Serializer
   include RoutingHelper
 
+  cache key: 'emoji', expires_in: 3.minutes
+
   context_extensions :emoji
 
   attributes :id, :type, :name, :updated
diff --git a/app/serializers/activitypub/note_serializer.rb b/app/serializers/activitypub/note_serializer.rb
index 67f596e78..87acc5429 100644
--- a/app/serializers/activitypub/note_serializer.rb
+++ b/app/serializers/activitypub/note_serializer.rb
@@ -1,6 +1,8 @@
 # frozen_string_literal: true
 
 class ActivityPub::NoteSerializer < ActivityPub::Serializer
+  cache key: 'note', expires_in: 3.minutes
+
   context_extensions :atom_uri, :conversation, :sensitive,
                      :hashtag, :emoji, :focal_point, :blurhash
 
diff --git a/app/services/process_hashtags_service.rb b/app/services/process_hashtags_service.rb
index d5ec076a8..b6974e598 100644
--- a/app/services/process_hashtags_service.rb
+++ b/app/services/process_hashtags_service.rb
@@ -14,7 +14,7 @@ class ProcessHashtagsService < BaseService
       TrendingTags.record_use!(tag, status.account, status.created_at) if status.public_visibility?
     end
 
-    return unless status.public_visibility? || status.unlisted_visibility?
+    return unless status.distributable?
 
     status.account.featured_tags.where(tag_id: records.map(&:id)).each do |featured_tag|
       featured_tag.increment(status.created_at)
diff --git a/app/views/statuses/_simple_status.html.haml b/app/views/statuses/_simple_status.html.haml
index 11220dfcb..38fde1be8 100644
--- a/app/views/statuses/_simple_status.html.haml
+++ b/app/views/statuses/_simple_status.html.haml
@@ -50,9 +50,9 @@
           = fa_icon 'reply-all fw'
       .status__action-bar__counter__label= obscured_counter status.replies_count
     = link_to remote_interaction_path(status, type: :reblog), class: 'status__action-bar-button icon-button modal-button', style: 'font-size: 18px; width: 23.1429px; height: 23.1429px; line-height: 23.15px;' do
-      - if status.public_visibility? || status.unlisted_visibility?
+      - if status.distributable?
         = fa_icon 'retweet fw'
-      - elsif status.private_visibility?
+      - elsif status.private_visibility? || status.limited_visibility?
         = fa_icon 'lock fw'
       - else
         = fa_icon 'envelope fw'
diff --git a/config/routes.rb b/config/routes.rb
index 69b495a96..115e7bb44 100644
--- a/config/routes.rb
+++ b/config/routes.rb
@@ -52,8 +52,9 @@ Rails.application.routes.draw do
       member do
         get :activity
         get :embed
-        get :replies
       end
+
+      resources :replies, only: [:index], module: :activitypub
     end
 
     resources :followers, only: [:index], controller: :follower_accounts
diff --git a/spec/controllers/concerns/account_controller_concern_spec.rb b/spec/controllers/concerns/account_controller_concern_spec.rb
index ea2b4a2a1..7ea214a7d 100644
--- a/spec/controllers/concerns/account_controller_concern_spec.rb
+++ b/spec/controllers/concerns/account_controller_concern_spec.rb
@@ -41,7 +41,7 @@ describe ApplicationController, type: :controller do
     it 'sets link headers' do
       account = Fabricate(:account, username: 'username', user: Fabricate(:user))
       get 'success', params: { account_username: 'username' }
-      expect(response.headers['Link'].to_s).to eq '<http://test.host/.well-known/webfinger?resource=acct%3Ausername%40cb6e6126.ngrok.io>; rel="lrdd"; type="application/xrd+xml", <http://test.host/users/username.atom>; rel="alternate"; type="application/atom+xml", <https://cb6e6126.ngrok.io/users/username>; rel="alternate"; type="application/activity+json"'
+      expect(response.headers['Link'].to_s).to eq '<http://test.host/.well-known/webfinger?resource=acct%3Ausername%40cb6e6126.ngrok.io>; rel="lrdd"; type="application/jrd+json", <https://cb6e6126.ngrok.io/users/username>; rel="alternate"; type="application/activity+json"'
     end
 
     it 'returns http success' do
diff --git a/spec/controllers/statuses_controller_spec.rb b/spec/controllers/statuses_controller_spec.rb
index 95e5c363c..6905dae10 100644
--- a/spec/controllers/statuses_controller_spec.rb
+++ b/spec/controllers/statuses_controller_spec.rb
@@ -92,7 +92,7 @@ describe StatusesController do
       end
 
       it 'assigns @max_descendant_thread_id for the last thread if it is hitting the status limit' do
-        stub_const 'StatusesController::DESCENDANTS_LIMIT', 1
+        stub_const 'StatusControllerConcern::DESCENDANTS_LIMIT', 1
         status = Fabricate(:status)
         child = Fabricate(:status, in_reply_to_id: status.id)
 
@@ -103,7 +103,7 @@ describe StatusesController do
       end
 
       it 'assigns @descendant_threads for threads with :next_status key if they are hitting the depth limit' do
-        stub_const 'StatusesController::DESCENDANTS_DEPTH_LIMIT', 2
+        stub_const 'StatusControllerConcern::DESCENDANTS_DEPTH_LIMIT', 2
         status = Fabricate(:status)
         child0 = Fabricate(:status, in_reply_to_id: status.id)
         child1 = Fabricate(:status, in_reply_to_id: child0.id)
diff --git a/spec/requests/link_headers_spec.rb b/spec/requests/link_headers_spec.rb
index 3dc408d92..712ee262b 100644
--- a/spec/requests/link_headers_spec.rb
+++ b/spec/requests/link_headers_spec.rb
@@ -11,16 +11,16 @@ describe 'Link headers' do
     end
 
     it 'contains webfinger url in link header' do
-      link_header = link_header_with_type('application/xrd+xml')
+      link_header = link_header_with_type('application/jrd+json')
 
       expect(link_header.href).to match 'http://www.example.com/.well-known/webfinger?resource=acct%3Atest%40cb6e6126.ngrok.io'
       expect(link_header.attr_pairs.first).to eq %w(rel lrdd)
     end
 
-    it 'contains atom url in link header' do
-      link_header = link_header_with_type('application/atom+xml')
+    it 'contains activitypub url in link header' do
+      link_header = link_header_with_type('application/activity+json')
 
-      expect(link_header.href).to eq 'http://www.example.com/users/test.atom'
+      expect(link_header.href).to eq 'https://cb6e6126.ngrok.io/users/test'
       expect(link_header.attr_pairs.first).to eq %w(rel alternate)
     end
 
-- 
cgit 


From ef1524639776fe7d7be2d5c414fc98dd2410a5f4 Mon Sep 17 00:00:00 2001
From: Eugen Rochko <eugen@zeonfederated.com>
Date: Mon, 8 Jul 2019 12:04:06 +0200
Subject: Remove unused remote unfollow controller (#11250)

---
 app/controllers/remote_unfollows_controller.rb     | 39 ----------------------
 app/views/remote_unfollows/_card.html.haml         | 13 --------
 .../_post_follow_actions.html.haml                 |  4 ---
 app/views/remote_unfollows/error.html.haml         |  3 --
 app/views/remote_unfollows/success.html.haml       | 10 ------
 config/locales/en.yml                              |  4 ---
 config/routes.rb                                   |  2 --
 .../remote_unfollows_controller_spec.rb            | 38 ---------------------
 8 files changed, 113 deletions(-)
 delete mode 100644 app/controllers/remote_unfollows_controller.rb
 delete mode 100644 app/views/remote_unfollows/_card.html.haml
 delete mode 100644 app/views/remote_unfollows/_post_follow_actions.html.haml
 delete mode 100644 app/views/remote_unfollows/error.html.haml
 delete mode 100644 app/views/remote_unfollows/success.html.haml
 delete mode 100644 spec/controllers/remote_unfollows_controller_spec.rb

(limited to 'spec')

diff --git a/app/controllers/remote_unfollows_controller.rb b/app/controllers/remote_unfollows_controller.rb
deleted file mode 100644
index af5943363..000000000
--- a/app/controllers/remote_unfollows_controller.rb
+++ /dev/null
@@ -1,39 +0,0 @@
-# frozen_string_literal: true
-
-class RemoteUnfollowsController < ApplicationController
-  layout 'modal'
-
-  before_action :authenticate_user!
-  before_action :set_body_classes
-
-  def create
-    @account = unfollow_attempt.try(:target_account)
-
-    if @account.nil?
-      render :error
-    else
-      render :success
-    end
-  rescue ActiveRecord::RecordNotFound, Mastodon::NotPermittedError
-    render :error
-  end
-
-  private
-
-  def unfollow_attempt
-    username, domain = acct_without_prefix.split('@')
-    UnfollowService.new.call(current_account, Account.find_remote!(username, domain))
-  end
-
-  def acct_without_prefix
-    acct_params.gsub(/\Aacct:/, '')
-  end
-
-  def acct_params
-    params.fetch(:acct, '')
-  end
-
-  def set_body_classes
-    @body_classes = 'modal-layout'
-  end
-end
diff --git a/app/views/remote_unfollows/_card.html.haml b/app/views/remote_unfollows/_card.html.haml
deleted file mode 100644
index 80ad3bae2..000000000
--- a/app/views/remote_unfollows/_card.html.haml
+++ /dev/null
@@ -1,13 +0,0 @@
-.account-card
-  .detailed-status__display-name
-    %div
-      = image_tag account.avatar.url(:original), alt: '', width: 48, height: 48, class: 'avatar'
-
-    %span.display-name
-      - account_url = local_assigns[:admin] ? admin_account_path(account.id) : ActivityPub::TagManager.instance.url_for(account)
-      = link_to account_url, class: 'detailed-status__display-name p-author h-card', target: '_blank', rel: 'noopener' do
-        %strong.emojify= display_name(account, custom_emojify: true)
-        %span @#{account.acct}
-
-  - if account.note?
-    .account__header__content.emojify= Formatter.instance.simplified_format(account)
diff --git a/app/views/remote_unfollows/_post_follow_actions.html.haml b/app/views/remote_unfollows/_post_follow_actions.html.haml
deleted file mode 100644
index 328f7c833..000000000
--- a/app/views/remote_unfollows/_post_follow_actions.html.haml
+++ /dev/null
@@ -1,4 +0,0 @@
-.post-follow-actions
-  %div= link_to t('authorize_follow.post_follow.web'), web_url("accounts/#{@account.id}"), class: 'button button--block'
-  %div= link_to t('authorize_follow.post_follow.return'), ActivityPub::TagManager.instance.url_for(@account), class: 'button button--block'
-  %div= t('authorize_follow.post_follow.close')
diff --git a/app/views/remote_unfollows/error.html.haml b/app/views/remote_unfollows/error.html.haml
deleted file mode 100644
index cb63f02be..000000000
--- a/app/views/remote_unfollows/error.html.haml
+++ /dev/null
@@ -1,3 +0,0 @@
-.form-container
-  .flash-message#error_explanation
-    = t('remote_unfollow.error')
diff --git a/app/views/remote_unfollows/success.html.haml b/app/views/remote_unfollows/success.html.haml
deleted file mode 100644
index b007eedc7..000000000
--- a/app/views/remote_unfollows/success.html.haml
+++ /dev/null
@@ -1,10 +0,0 @@
-- content_for :page_title do
-  = t('remote_unfollow.title', acct: @account.acct)
-
-.form-container
-  .follow-prompt
-    %h2= t('remote_unfollow.unfollowed')
-
-    = render 'application/card', account: @account
-
-  = render 'post_follow_actions'
diff --git a/config/locales/en.yml b/config/locales/en.yml
index 611f36fdd..00b7d1dbe 100644
--- a/config/locales/en.yml
+++ b/config/locales/en.yml
@@ -810,10 +810,6 @@ en:
     reply:
       proceed: Proceed to reply
       prompt: 'You want to reply to this toot:'
-  remote_unfollow:
-    error: Error
-    title: Title
-    unfollowed: Unfollowed
   scheduled_statuses:
     over_daily_limit: You have exceeded the limit of %{limit} scheduled toots for that day
     over_total_limit: You have exceeded the limit of %{limit} scheduled toots
diff --git a/config/routes.rb b/config/routes.rb
index 115e7bb44..95f8a39ad 100644
--- a/config/routes.rb
+++ b/config/routes.rb
@@ -141,8 +141,6 @@ Rails.application.routes.draw do
   get '/public', to: 'public_timelines#show', as: :public_timeline
   get '/media_proxy/:id/(*any)', to: 'media_proxy#show', as: :media_proxy
 
-  # Remote follow
-  resource :remote_unfollow, only: [:create]
   resource :authorize_interaction, only: [:show, :create]
   resource :share, only: [:show, :create]
 
diff --git a/spec/controllers/remote_unfollows_controller_spec.rb b/spec/controllers/remote_unfollows_controller_spec.rb
deleted file mode 100644
index a1a55ede0..000000000
--- a/spec/controllers/remote_unfollows_controller_spec.rb
+++ /dev/null
@@ -1,38 +0,0 @@
-# frozen_string_literal: true
-
-require 'rails_helper'
-
-describe RemoteUnfollowsController do
-  render_views
-
-  describe '#create' do
-    subject { post :create, params: { acct: acct } }
-
-    let(:current_user) { Fabricate(:user, account: current_account) }
-    let(:current_account) { Fabricate(:account) }
-    let(:remote_account) { Fabricate(:user, email: 'bob@example.com', account: Fabricate(:account, username: 'bob', protocol: :activitypub, domain: 'example.com', inbox_url: 'http://example.com/inbox')).account }
-    before do
-      sign_in current_user
-      current_account.follow!(remote_account)
-      stub_request(:post, 'http://example.com/inbox') { { status: 200 } }
-    end
-
-    context 'when successfully unfollow remote account' do
-      let(:acct) { "acct:#{remote_account.username}@#{remote_account.domain}" }
-
-      it do
-        is_expected.to render_template :success
-        expect(current_account.following?(remote_account)).to be false
-      end
-    end
-
-    context 'when fails to unfollow remote account' do
-      let(:acct) { "acct:#{remote_account.username + '_test'}@#{remote_account.domain}" }
-
-      it do
-        is_expected.to render_template :error
-        expect(current_account.following?(remote_account)).to be true
-      end
-    end
-  end
-end
-- 
cgit 


From 4e921832272425352d28cad550bfc4dffd6d0e78 Mon Sep 17 00:00:00 2001
From: Eugen Rochko <eugen@zeonfederated.com>
Date: Tue, 9 Jul 2019 03:27:35 +0200
Subject: Refactor domain block checks (#11268)

---
 app/controllers/concerns/signature_verification.rb |   4 +
 app/helpers/domain_control_helper.rb               |  17 ++++
 app/lib/tag_manager.rb                             |   3 +
 .../fetch_featured_collection_service.rb           |   3 +-
 .../activitypub/fetch_remote_account_service.rb    |  14 ++-
 .../activitypub/fetch_remote_poll_service.rb       |   2 +
 .../activitypub/process_account_service.rb         |   5 +-
 .../activitypub/process_collection_service.rb      |   4 +-
 app/services/activitypub/process_poll_service.rb   |   1 +
 app/services/resolve_account_service.rb            | 101 +++++++++++++--------
 spec/services/resolve_account_service_spec.rb      |   5 +
 11 files changed, 108 insertions(+), 51 deletions(-)
 create mode 100644 app/helpers/domain_control_helper.rb

(limited to 'spec')

diff --git a/app/controllers/concerns/signature_verification.rb b/app/controllers/concerns/signature_verification.rb
index 90a57197c..0ccdf5ec9 100644
--- a/app/controllers/concerns/signature_verification.rb
+++ b/app/controllers/concerns/signature_verification.rb
@@ -5,6 +5,8 @@
 module SignatureVerification
   extend ActiveSupport::Concern
 
+  include DomainControlHelper
+
   def signed_request?
     request.headers['Signature'].present?
   end
@@ -126,6 +128,8 @@ module SignatureVerification
     if key_id.start_with?('acct:')
       stoplight_wrap_request { ResolveAccountService.new.call(key_id.gsub(/\Aacct:/, '')) }
     elsif !ActivityPub::TagManager.instance.local_uri?(key_id)
+      return if domain_not_allowed?(key_id)
+
       account   = ActivityPub::TagManager.instance.uri_to_resource(key_id, Account)
       account ||= stoplight_wrap_request { ActivityPub::FetchRemoteKeyService.new.call(key_id, id: false) }
       account
diff --git a/app/helpers/domain_control_helper.rb b/app/helpers/domain_control_helper.rb
new file mode 100644
index 000000000..efd328f81
--- /dev/null
+++ b/app/helpers/domain_control_helper.rb
@@ -0,0 +1,17 @@
+# frozen_string_literal: true
+
+module DomainControlHelper
+  def domain_not_allowed?(uri_or_domain)
+    return if uri_or_domain.blank?
+
+    domain = begin
+      if uri_or_domain.include?('://')
+        Addressable::URI.parse(uri_or_domain).domain
+      else
+        uri_or_domain
+      end
+    end
+
+    DomainBlock.blocked?(domain)
+  end
+end
diff --git a/app/lib/tag_manager.rb b/app/lib/tag_manager.rb
index daf4f556b..c88cf4994 100644
--- a/app/lib/tag_manager.rb
+++ b/app/lib/tag_manager.rb
@@ -24,13 +24,16 @@ class TagManager
 
   def same_acct?(canonical, needle)
     return true if canonical.casecmp(needle).zero?
+
     username, domain = needle.split('@')
+
     local_domain?(domain) && canonical.casecmp(username).zero?
   end
 
   def local_url?(url)
     uri    = Addressable::URI.parse(url).normalize
     domain = uri.host + (uri.port ? ":#{uri.port}" : '')
+
     TagManager.instance.web_domain?(domain)
   end
 end
diff --git a/app/services/activitypub/fetch_featured_collection_service.rb b/app/services/activitypub/fetch_featured_collection_service.rb
index 6a137b520..2c2770466 100644
--- a/app/services/activitypub/fetch_featured_collection_service.rb
+++ b/app/services/activitypub/fetch_featured_collection_service.rb
@@ -4,13 +4,12 @@ class ActivityPub::FetchFeaturedCollectionService < BaseService
   include JsonLdHelper
 
   def call(account)
-    return if account.featured_collection_url.blank?
+    return if account.featured_collection_url.blank? || account.suspended? || account.local?
 
     @account = account
     @json    = fetch_resource(@account.featured_collection_url, true)
 
     return unless supported_context?
-    return if @account.suspended? || @account.local?
 
     case @json['type']
     when 'Collection', 'CollectionPage'
diff --git a/app/services/activitypub/fetch_remote_account_service.rb b/app/services/activitypub/fetch_remote_account_service.rb
index 3c2044941..d65c8f951 100644
--- a/app/services/activitypub/fetch_remote_account_service.rb
+++ b/app/services/activitypub/fetch_remote_account_service.rb
@@ -2,18 +2,22 @@
 
 class ActivityPub::FetchRemoteAccountService < BaseService
   include JsonLdHelper
+  include DomainControlHelper
 
   SUPPORTED_TYPES = %w(Application Group Organization Person Service).freeze
 
   # Does a WebFinger roundtrip on each call, unless `only_key` is true
   def call(uri, id: true, prefetched_body: nil, break_on_redirect: false, only_key: false)
+    return if domain_not_allowed?(uri)
     return ActivityPub::TagManager.instance.uri_to_resource(uri, Account) if ActivityPub::TagManager.instance.local_uri?(uri)
 
-    @json = if prefetched_body.nil?
-              fetch_resource(uri, id)
-            else
-              body_to_json(prefetched_body, compare_id: id ? uri : nil)
-            end
+    @json = begin
+      if prefetched_body.nil?
+        fetch_resource(uri, id)
+      else
+        body_to_json(prefetched_body, compare_id: id ? uri : nil)
+      end
+    end
 
     return if !supported_context? || !expected_type? || (break_on_redirect && @json['movedTo'].present?)
 
diff --git a/app/services/activitypub/fetch_remote_poll_service.rb b/app/services/activitypub/fetch_remote_poll_service.rb
index 854a32d05..1c79ecf11 100644
--- a/app/services/activitypub/fetch_remote_poll_service.rb
+++ b/app/services/activitypub/fetch_remote_poll_service.rb
@@ -5,7 +5,9 @@ class ActivityPub::FetchRemotePollService < BaseService
 
   def call(poll, on_behalf_of = nil)
     json = fetch_resource(poll.status.uri, true, on_behalf_of)
+
     return unless supported_context?(json)
+
     ActivityPub::ProcessPollService.new.call(poll, json)
   end
 end
diff --git a/app/services/activitypub/process_account_service.rb b/app/services/activitypub/process_account_service.rb
index 3857e7c16..603e27ed9 100644
--- a/app/services/activitypub/process_account_service.rb
+++ b/app/services/activitypub/process_account_service.rb
@@ -2,11 +2,12 @@
 
 class ActivityPub::ProcessAccountService < BaseService
   include JsonLdHelper
+  include DomainControlHelper
 
   # Should be called with confirmed valid JSON
   # and WebFinger-resolved username and domain
   def call(username, domain, json, options = {})
-    return if json['inbox'].blank? || unsupported_uri_scheme?(json['id'])
+    return if json['inbox'].blank? || unsupported_uri_scheme?(json['id']) || domain_not_allowed?(domain)
 
     @options     = options
     @json        = json
@@ -15,8 +16,6 @@ class ActivityPub::ProcessAccountService < BaseService
     @domain      = domain
     @collections = {}
 
-    return if auto_suspend?
-
     RedisLock.acquire(lock_options) do |lock|
       if lock.acquired?
         @account        = Account.find_remote(@username, @domain)
diff --git a/app/services/activitypub/process_collection_service.rb b/app/services/activitypub/process_collection_service.rb
index 881df478b..a2a2e7071 100644
--- a/app/services/activitypub/process_collection_service.rb
+++ b/app/services/activitypub/process_collection_service.rb
@@ -8,9 +8,7 @@ class ActivityPub::ProcessCollectionService < BaseService
     @json    = Oj.load(body, mode: :strict)
     @options = options
 
-    return unless supported_context?
-    return if different_actor? && verify_account!.nil?
-    return if @account.suspended? || @account.local?
+    return if !supported_context? || (different_actor? && verify_account!.nil?) || @account.suspended? || @account.local?
 
     case @json['type']
     when 'Collection', 'CollectionPage'
diff --git a/app/services/activitypub/process_poll_service.rb b/app/services/activitypub/process_poll_service.rb
index 61357abd3..2fbce65b9 100644
--- a/app/services/activitypub/process_poll_service.rb
+++ b/app/services/activitypub/process_poll_service.rb
@@ -5,6 +5,7 @@ class ActivityPub::ProcessPollService < BaseService
 
   def call(poll, json)
     @json = json
+
     return unless expected_type?
 
     previous_expires_at = poll.expires_at
diff --git a/app/services/resolve_account_service.rb b/app/services/resolve_account_service.rb
index 0ea31a0d8..41a2eb158 100644
--- a/app/services/resolve_account_service.rb
+++ b/app/services/resolve_account_service.rb
@@ -1,75 +1,108 @@
 # frozen_string_literal: true
 
-require_relative '../models/account'
-
 class ResolveAccountService < BaseService
   include JsonLdHelper
+  include DomainControlHelper
+
+  class WebfingerRedirectError < StandardError; end
 
-  # Find or create a local account for a remote user.
-  # When creating, look up the user's webfinger and fetch all
-  # important information from their feed
-  # @param [String, Account] uri User URI in the form of username@domain
+  # Find or create an account record for a remote user. When creating,
+  # look up the user's webfinger and fetch ActivityPub data
+  # @param [String, Account] uri URI in the username@domain format or account record
   # @param [Hash] options
+  # @option options [Boolean] :redirected Do not follow further Webfinger redirects
+  # @option options [Boolean] :skip_webfinger Do not attempt to refresh account data
   # @return [Account]
   def call(uri, options = {})
+    return if uri.blank?
+
+    process_options!(uri, options)
+
+    # First of all we want to check if we've got the account
+    # record with the URI already, and if so, we can exit early
+
+    return if domain_not_allowed?(@domain)
+
+    @account ||= Account.find_remote(@username, @domain)
+
+    return @account if @account&.local? || !webfinger_update_due?
+
+    # At this point we are in need of a Webfinger query, which may
+    # yield us a different username/domain through a redirect
+
+    process_webfinger!
+
+    # Because the username/domain pair may be different than what
+    # we already checked, we need to check if we've already got
+    # the record with that URI, again
+
+    return if domain_not_allowed?(@domain)
+
+    @account ||= Account.find_remote(@username, @domain)
+
+    return @account if @account&.local? || !webfinger_update_due?
+
+    # Now it is certain, it is definitely a remote account, and it
+    # either needs to be created, or updated from fresh data
+
+    process_account!
+  rescue Goldfinger::Error, WebfingerRedirectError, Oj::ParseError => e
+    Rails.logger.debug "Webfinger query for #{@uri} failed: #{e}"
+    nil
+  end
+
+  private
+
+  def process_options!(uri, options)
     @options = options
 
     if uri.is_a?(Account)
       @account  = uri
       @username = @account.username
       @domain   = @account.domain
-      uri       = "#{@username}@#{@domain}"
-
-      return @account if @account.local? || !webfinger_update_due?
+      @uri      = [@username, @domain].compact.join('@')
     else
+      @uri               = uri
       @username, @domain = uri.split('@')
-
-      return Account.find_local(@username) if TagManager.instance.local_domain?(@domain)
-
-      @account = Account.find_remote(@username, @domain)
-
-      return @account unless webfinger_update_due?
     end
 
-    Rails.logger.debug "Looking up webfinger for #{uri}"
-
-    @webfinger = Goldfinger.finger("acct:#{uri}")
+    @domain = nil if TagManager.instance.local_domain?(@domain)
+  end
 
+  def process_webfinger!
+    @webfinger                           = Goldfinger.finger("acct:#{@uri}")
     confirmed_username, confirmed_domain = @webfinger.subject.gsub(/\Aacct:/, '').split('@')
 
     if confirmed_username.casecmp(@username).zero? && confirmed_domain.casecmp(@domain).zero?
       @username = confirmed_username
       @domain   = confirmed_domain
-    elsif options[:redirected].nil?
-      return call("#{confirmed_username}@#{confirmed_domain}", options.merge(redirected: true))
+    elsif @options[:redirected].nil?
+      @account = ResolveAccountService.new.call("#{confirmed_username}@#{confirmed_domain}", @options.merge(redirected: true))
     else
-      Rails.logger.debug 'Requested and returned acct URIs do not match'
-      return
+      raise WebfingerRedirectError, "The URI #{uri} tries to hijack #{@username}@#{@domain}"
     end
 
-    return Account.find_local(@username) if TagManager.instance.local_domain?(@domain)
+    @domain = nil if TagManager.instance.local_domain?(@domain)
+  end
+
+  def process_account!
     return unless activitypub_ready?
 
     RedisLock.acquire(lock_options) do |lock|
       if lock.acquired?
         @account = Account.find_remote(@username, @domain)
 
-        next unless @account.nil? || @account.activitypub?
+        next if (@account.present? && !@account.activitypub?) || actor_json.nil?
 
-        handle_activitypub
+        @account = ActivityPub::ProcessAccountService.new.call(@username, @domain, actor_json)
       else
         raise Mastodon::RaceConditionError
       end
     end
 
     @account
-  rescue Goldfinger::Error => e
-    Rails.logger.debug "Webfinger query for #{uri} unsuccessful: #{e}"
-    nil
   end
 
-  private
-
   def webfinger_update_due?
     @account.nil? || ((!@options[:skip_webfinger] || @account.ostatus?) && @account.possibly_stale?)
   end
@@ -78,14 +111,6 @@ class ResolveAccountService < BaseService
     !@webfinger.link('self').nil? && ['application/activity+json', 'application/ld+json; profile="https://www.w3.org/ns/activitystreams"'].include?(@webfinger.link('self').type)
   end
 
-  def handle_activitypub
-    return if actor_json.nil?
-
-    @account = ActivityPub::ProcessAccountService.new.call(@username, @domain, actor_json)
-  rescue Oj::ParseError
-    nil
-  end
-
   def actor_url
     @actor_url ||= @webfinger.link('self').href
   end
diff --git a/spec/services/resolve_account_service_spec.rb b/spec/services/resolve_account_service_spec.rb
index 7a64f4161..cea942e39 100644
--- a/spec/services/resolve_account_service_spec.rb
+++ b/spec/services/resolve_account_service_spec.rb
@@ -53,6 +53,11 @@ RSpec.describe ResolveAccountService, type: :service do
     fail_occurred  = false
     return_values  = Concurrent::Array.new
 
+    # Preload classes that throw circular dependency errors in threads
+    Account
+    TagManager
+    DomainBlock
+
     threads = Array.new(5) do
       Thread.new do
         true while wait_for_start
-- 
cgit 


From 5d3feed191bcbe2769512119752b426108152fe9 Mon Sep 17 00:00:00 2001
From: Eugen Rochko <eugen@zeonfederated.com>
Date: Wed, 10 Jul 2019 18:59:28 +0200
Subject: Refactor fetching of remote resources (#11251)

---
 app/helpers/jsonld_helper.rb                       | 47 ++++++-----
 app/lib/request.rb                                 |  2 +-
 .../activitypub/fetch_remote_status_service.rb     | 20 ++---
 app/services/fetch_atom_service.rb                 | 93 ---------------------
 app/services/fetch_link_card_service.rb            |  2 +-
 app/services/fetch_remote_account_service.rb       |  2 +-
 app/services/fetch_remote_status_service.rb        |  2 +-
 app/services/fetch_resource_service.rb             | 68 +++++++++++++++
 app/services/resolve_url_service.rb                | 47 ++++-------
 app/workers/activitypub/delivery_worker.rb         | 16 ++--
 spec/services/fetch_atom_service_spec.rb           | 96 ----------------------
 spec/services/fetch_remote_account_service_spec.rb |  1 +
 spec/services/fetch_resource_service_spec.rb       | 96 ++++++++++++++++++++++
 spec/services/resolve_url_service_spec.rb          | 44 ++--------
 14 files changed, 231 insertions(+), 305 deletions(-)
 delete mode 100644 app/services/fetch_atom_service.rb
 create mode 100644 app/services/fetch_resource_service.rb
 delete mode 100644 spec/services/fetch_atom_service_spec.rb
 create mode 100644 spec/services/fetch_resource_service_spec.rb

(limited to 'spec')

diff --git a/app/helpers/jsonld_helper.rb b/app/helpers/jsonld_helper.rb
index 5b4011275..34a657e06 100644
--- a/app/helpers/jsonld_helper.rb
+++ b/app/helpers/jsonld_helper.rb
@@ -16,13 +16,15 @@ module JsonLdHelper
   # The url attribute can be a string, an array of strings, or an array of objects.
   # The objects could include a mimeType. Not-included mimeType means it's text/html.
   def url_to_href(value, preferred_type = nil)
-    single_value = if value.is_a?(Array) && !value.first.is_a?(String)
-                     value.find { |link| preferred_type.nil? || ((link['mimeType'].presence || 'text/html') == preferred_type) }
-                   elsif value.is_a?(Array)
-                     value.first
-                   else
-                     value
-                   end
+    single_value = begin
+      if value.is_a?(Array) && !value.first.is_a?(String)
+        value.find { |link| preferred_type.nil? || ((link['mimeType'].presence || 'text/html') == preferred_type) }
+      elsif value.is_a?(Array)
+        value.first
+      else
+        value
+      end
+    end
 
     if single_value.nil? || single_value.is_a?(String)
       single_value
@@ -64,7 +66,9 @@ module JsonLdHelper
   def fetch_resource(uri, id, on_behalf_of = nil)
     unless id
       json = fetch_resource_without_id_validation(uri, on_behalf_of)
+
       return unless json
+
       uri = json['id']
     end
 
@@ -74,24 +78,26 @@ module JsonLdHelper
 
   def fetch_resource_without_id_validation(uri, on_behalf_of = nil, raise_on_temporary_error = false)
     build_request(uri, on_behalf_of).perform do |response|
-      unless response_successful?(response) || response_error_unsalvageable?(response) || !raise_on_temporary_error
-        raise Mastodon::UnexpectedResponseError, response
-      end
+      raise Mastodon::UnexpectedResponseError, response unless response_successful?(response) || response_error_unsalvageable?(response) || !raise_on_temporary_error
+
       return body_to_json(response.body_with_limit) if response.code == 200
     end
+
     # If request failed, retry without doing it on behalf of a user
     return if on_behalf_of.nil?
+
     build_request(uri).perform do |response|
-      unless response_successful?(response) || response_error_unsalvageable?(response) || !raise_on_temporary_error
-        raise Mastodon::UnexpectedResponseError, response
-      end
+      raise Mastodon::UnexpectedResponseError, response unless response_successful?(response) || response_error_unsalvageable?(response) || !raise_on_temporary_error
+
       response.code == 200 ? body_to_json(response.body_with_limit) : nil
     end
   end
 
   def body_to_json(body, compare_id: nil)
     json = body.is_a?(String) ? Oj.load(body, mode: :strict) : body
+
     return if compare_id.present? && json['id'] != compare_id
+
     json
   rescue Oj::ParseError
     nil
@@ -105,35 +111,34 @@ module JsonLdHelper
     end
   end
 
-  private
-
   def response_successful?(response)
     (200...300).cover?(response.code)
   end
 
   def response_error_unsalvageable?(response)
-    (400...500).cover?(response.code) && response.code != 429
+    response.code == 501 || ((400...500).cover?(response.code) && ![401, 408, 429].include?(response.code))
   end
 
   def build_request(uri, on_behalf_of = nil)
-    request = Request.new(:get, uri)
-    request.on_behalf_of(on_behalf_of) if on_behalf_of
-    request.add_headers('Accept' => 'application/activity+json, application/ld+json')
-    request
+    Request.new(:get, uri).tap do |request|
+      request.on_behalf_of(on_behalf_of) if on_behalf_of
+      request.add_headers('Accept' => 'application/activity+json, application/ld+json')
+    end
   end
 
   def load_jsonld_context(url, _options = {}, &_block)
     json = Rails.cache.fetch("jsonld:context:#{url}", expires_in: 30.days, raw: true) do
       request = Request.new(:get, url)
       request.add_headers('Accept' => 'application/ld+json')
-
       request.perform do |res|
         raise JSON::LD::JsonLdError::LoadingDocumentFailed unless res.code == 200 && res.mime_type == 'application/ld+json'
+
         res.body_with_limit
       end
     end
 
     doc = JSON::LD::API::RemoteDocument.new(url, json)
+
     block_given? ? yield(doc) : doc
   end
 end
diff --git a/app/lib/request.rb b/app/lib/request.rb
index 322457ad7..1fd3f5190 100644
--- a/app/lib/request.rb
+++ b/app/lib/request.rb
@@ -41,7 +41,7 @@ class Request
   end
 
   def on_behalf_of(account, key_id_format = :acct, sign_with: nil)
-    raise ArgumentError unless account.local?
+    raise ArgumentError, 'account must be local' unless account&.local?
 
     @account       = account
     @keypair       = sign_with.present? ? OpenSSL::PKey::RSA.new(sign_with) : @account.keypair
diff --git a/app/services/activitypub/fetch_remote_status_service.rb b/app/services/activitypub/fetch_remote_status_service.rb
index 469821032..cf4f62899 100644
--- a/app/services/activitypub/fetch_remote_status_service.rb
+++ b/app/services/activitypub/fetch_remote_status_service.rb
@@ -5,18 +5,18 @@ class ActivityPub::FetchRemoteStatusService < BaseService
 
   # Should be called when uri has already been checked for locality
   def call(uri, id: true, prefetched_body: nil, on_behalf_of: nil)
-    @json = if prefetched_body.nil?
-              fetch_resource(uri, id, on_behalf_of)
-            else
-              body_to_json(prefetched_body, compare_id: id ? uri : nil)
-            end
+    @json = begin
+      if prefetched_body.nil?
+        fetch_resource(uri, id, on_behalf_of)
+      else
+        body_to_json(prefetched_body, compare_id: id ? uri : nil)
+      end
+    end
 
-    return unless supported_context? && expected_type?
-
-    return if actor_id.nil? || !trustworthy_attribution?(@json['id'], actor_id)
+    return if !(supported_context? && expected_type?) || actor_id.nil? || !trustworthy_attribution?(@json['id'], actor_id)
 
     actor = ActivityPub::TagManager.instance.uri_to_resource(actor_id, Account)
-    actor = ActivityPub::FetchRemoteAccountService.new.call(actor_id, id: true) if actor.nil? || needs_update(actor)
+    actor = ActivityPub::FetchRemoteAccountService.new.call(actor_id, id: true) if actor.nil? || needs_update?(actor)
 
     return if actor.nil? || actor.suspended?
 
@@ -46,7 +46,7 @@ class ActivityPub::FetchRemoteStatusService < BaseService
     equals_or_includes_any?(@json['type'], ActivityPub::Activity::Create::SUPPORTED_TYPES + ActivityPub::Activity::Create::CONVERTED_TYPES)
   end
 
-  def needs_update(actor)
+  def needs_update?(actor)
     actor.possibly_stale?
   end
 end
diff --git a/app/services/fetch_atom_service.rb b/app/services/fetch_atom_service.rb
deleted file mode 100644
index d6508a988..000000000
--- a/app/services/fetch_atom_service.rb
+++ /dev/null
@@ -1,93 +0,0 @@
-# frozen_string_literal: true
-
-class FetchAtomService < BaseService
-  include JsonLdHelper
-
-  def call(url)
-    return if url.blank?
-
-    result = process(url)
-
-    # retry without ActivityPub
-    result ||= process(url) if @unsupported_activity
-
-    result
-  rescue OpenSSL::SSL::SSLError => e
-    Rails.logger.debug "SSL error: #{e}"
-    nil
-  rescue HTTP::ConnectionError => e
-    Rails.logger.debug "HTTP ConnectionError: #{e}"
-    nil
-  end
-
-  private
-
-  def process(url, terminal = false)
-    @url = url
-    perform_request { |response| process_response(response, terminal) }
-  end
-
-  def perform_request(&block)
-    accept = 'text/html'
-    accept = 'application/activity+json, application/ld+json; profile="https://www.w3.org/ns/activitystreams", application/atom+xml, ' + accept unless @unsupported_activity
-
-    Request.new(:get, @url).add_headers('Accept' => accept).perform(&block)
-  end
-
-  def process_response(response, terminal = false)
-    return nil if response.code != 200
-
-    if response.mime_type == 'application/atom+xml'
-      [@url, { prefetched_body: response.body_with_limit }, :ostatus]
-    elsif ['application/activity+json', 'application/ld+json'].include?(response.mime_type)
-      body = response.body_with_limit
-      json = body_to_json(body)
-      if supported_context?(json) && equals_or_includes_any?(json['type'], ActivityPub::FetchRemoteAccountService::SUPPORTED_TYPES) && json['inbox'].present?
-        [json['id'], { prefetched_body: body, id: true }, :activitypub]
-      elsif supported_context?(json) && expected_type?(json)
-        [json['id'], { prefetched_body: body, id: true }, :activitypub]
-      else
-        @unsupported_activity = true
-        nil
-      end
-    elsif !terminal
-      link_header = response['Link'] && parse_link_header(response)
-
-      if link_header&.find_link(%w(rel alternate))
-        process_link_headers(link_header)
-      elsif response.mime_type == 'text/html'
-        process_html(response)
-      end
-    end
-  end
-
-  def expected_type?(json)
-    equals_or_includes_any?(json['type'], ActivityPub::Activity::Create::SUPPORTED_TYPES + ActivityPub::Activity::Create::CONVERTED_TYPES)
-  end
-
-  def process_html(response)
-    page = Nokogiri::HTML(response.body_with_limit)
-
-    json_link = page.xpath('//link[@rel="alternate"]').find { |link| ['application/activity+json', 'application/ld+json; profile="https://www.w3.org/ns/activitystreams"'].include?(link['type']) }
-    atom_link = page.xpath('//link[@rel="alternate"]').find { |link| link['type'] == 'application/atom+xml' }
-
-    result ||= process(json_link['href'], terminal: true) unless json_link.nil? || @unsupported_activity
-    result ||= process(atom_link['href'], terminal: true) unless atom_link.nil?
-
-    result
-  end
-
-  def process_link_headers(link_header)
-    json_link = link_header.find_link(%w(rel alternate), %w(type application/activity+json)) || link_header.find_link(%w(rel alternate), ['type', 'application/ld+json; profile="https://www.w3.org/ns/activitystreams"'])
-    atom_link = link_header.find_link(%w(rel alternate), %w(type application/atom+xml))
-
-    result ||= process(json_link.href, terminal: true) unless json_link.nil? || @unsupported_activity
-    result ||= process(atom_link.href, terminal: true) unless atom_link.nil?
-
-    result
-  end
-
-  def parse_link_header(response)
-    LinkHeader.parse(response['Link'].is_a?(Array) ? response['Link'].first : response['Link'])
-  end
-end
diff --git a/app/services/fetch_link_card_service.rb b/app/services/fetch_link_card_service.rb
index 75fbd0e8c..4e75c370f 100644
--- a/app/services/fetch_link_card_service.rb
+++ b/app/services/fetch_link_card_service.rb
@@ -29,7 +29,7 @@ class FetchLinkCardService < BaseService
     end
 
     attach_card if @card&.persisted?
-  rescue HTTP::Error, Addressable::URI::InvalidURIError, Mastodon::HostValidationError, Mastodon::LengthValidationError => e
+  rescue HTTP::Error, OpenSSL::SSL::SSLError, Addressable::URI::InvalidURIError, Mastodon::HostValidationError, Mastodon::LengthValidationError => e
     Rails.logger.debug "Error fetching link #{@url}: #{e}"
     nil
   end
diff --git a/app/services/fetch_remote_account_service.rb b/app/services/fetch_remote_account_service.rb
index a7f95603d..3cd06e30f 100644
--- a/app/services/fetch_remote_account_service.rb
+++ b/app/services/fetch_remote_account_service.rb
@@ -3,7 +3,7 @@
 class FetchRemoteAccountService < BaseService
   def call(url, prefetched_body = nil, protocol = :ostatus)
     if prefetched_body.nil?
-      resource_url, resource_options, protocol = FetchAtomService.new.call(url)
+      resource_url, resource_options, protocol = FetchResourceService.new.call(url)
     else
       resource_url     = url
       resource_options = { prefetched_body: prefetched_body }
diff --git a/app/services/fetch_remote_status_service.rb b/app/services/fetch_remote_status_service.rb
index aac39dfd5..208dc7809 100644
--- a/app/services/fetch_remote_status_service.rb
+++ b/app/services/fetch_remote_status_service.rb
@@ -3,7 +3,7 @@
 class FetchRemoteStatusService < BaseService
   def call(url, prefetched_body = nil, protocol = :ostatus)
     if prefetched_body.nil?
-      resource_url, resource_options, protocol = FetchAtomService.new.call(url)
+      resource_url, resource_options, protocol = FetchResourceService.new.call(url)
     else
       resource_url     = url
       resource_options = { prefetched_body: prefetched_body }
diff --git a/app/services/fetch_resource_service.rb b/app/services/fetch_resource_service.rb
new file mode 100644
index 000000000..c0473f3ad
--- /dev/null
+++ b/app/services/fetch_resource_service.rb
@@ -0,0 +1,68 @@
+# frozen_string_literal: true
+
+class FetchResourceService < BaseService
+  include JsonLdHelper
+
+  ACCEPT_HEADER = 'application/activity+json, application/ld+json; profile="https://www.w3.org/ns/activitystreams", text/html'
+
+  def call(url)
+    return if url.blank?
+
+    process(url)
+  rescue HTTP::Error, OpenSSL::SSL::SSLError, Addressable::URI::InvalidURIError, Mastodon::HostValidationError, Mastodon::LengthValidationError => e
+    Rails.logger.debug "Error fetching resource #{@url}: #{e}"
+    nil
+  end
+
+  private
+
+  def process(url, terminal = false)
+    @url = url
+
+    perform_request { |response| process_response(response, terminal) }
+  end
+
+  def perform_request(&block)
+    Request.new(:get, @url).add_headers('Accept' => ACCEPT_HEADER).perform(&block)
+  end
+
+  def process_response(response, terminal = false)
+    return nil if response.code != 200
+
+    if ['application/activity+json', 'application/ld+json'].include?(response.mime_type)
+      body = response.body_with_limit
+      json = body_to_json(body)
+
+      [json['id'], { prefetched_body: body, id: true }, :activitypub] if supported_context?(json) && (equals_or_includes_any?(json['type'], ActivityPub::FetchRemoteAccountService::SUPPORTED_TYPES) || expected_type?(json))
+    elsif !terminal
+      link_header = response['Link'] && parse_link_header(response)
+
+      if link_header&.find_link(%w(rel alternate))
+        process_link_headers(link_header)
+      elsif response.mime_type == 'text/html'
+        process_html(response)
+      end
+    end
+  end
+
+  def expected_type?(json)
+    equals_or_includes_any?(json['type'], ActivityPub::Activity::Create::SUPPORTED_TYPES + ActivityPub::Activity::Create::CONVERTED_TYPES)
+  end
+
+  def process_html(response)
+    page      = Nokogiri::HTML(response.body_with_limit)
+    json_link = page.xpath('//link[@rel="alternate"]').find { |link| ['application/activity+json', 'application/ld+json; profile="https://www.w3.org/ns/activitystreams"'].include?(link['type']) }
+
+    process(json_link['href'], terminal: true) unless json_link.nil?
+  end
+
+  def process_link_headers(link_header)
+    json_link = link_header.find_link(%w(rel alternate), %w(type application/activity+json)) || link_header.find_link(%w(rel alternate), ['type', 'application/ld+json; profile="https://www.w3.org/ns/activitystreams"'])
+
+    process(json_link.href, terminal: true) unless json_link.nil?
+  end
+
+  def parse_link_header(response)
+    LinkHeader.parse(response['Link'].is_a?(Array) ? response['Link'].first : response['Link'])
+  end
+end
diff --git a/app/services/resolve_url_service.rb b/app/services/resolve_url_service.rb
index f941b489a..80381c16b 100644
--- a/app/services/resolve_url_service.rb
+++ b/app/services/resolve_url_service.rb
@@ -4,64 +4,49 @@ class ResolveURLService < BaseService
   include JsonLdHelper
   include Authorization
 
-  attr_reader :url
-
   def call(url, on_behalf_of: nil)
-    @url = url
+    @url          = url
     @on_behalf_of = on_behalf_of
 
-    return process_local_url if local_url?
-
-    process_url unless fetched_atom_feed.nil?
+    if local_url?
+      process_local_url
+    elsif !fetched_resource.nil?
+      process_url
+    end
   end
 
   private
 
   def process_url
     if equals_or_includes_any?(type, ActivityPub::FetchRemoteAccountService::SUPPORTED_TYPES)
-      FetchRemoteAccountService.new.call(atom_url, body, protocol)
+      FetchRemoteAccountService.new.call(resource_url, body, protocol)
     elsif equals_or_includes_any?(type, ActivityPub::Activity::Create::SUPPORTED_TYPES + ActivityPub::Activity::Create::CONVERTED_TYPES)
-      FetchRemoteStatusService.new.call(atom_url, body, protocol)
+      FetchRemoteStatusService.new.call(resource_url, body, protocol)
     end
   end
 
-  def fetched_atom_feed
-    @_fetched_atom_feed ||= FetchAtomService.new.call(url)
+  def fetched_resource
+    @fetched_resource ||= FetchResourceService.new.call(@url)
   end
 
-  def atom_url
-    fetched_atom_feed.first
+  def resource_url
+    fetched_resource.first
   end
 
   def body
-    fetched_atom_feed.second[:prefetched_body]
+    fetched_resource.second[:prefetched_body]
   end
 
   def protocol
-    fetched_atom_feed.third
+    fetched_resource.third
   end
 
   def type
     return json_data['type'] if protocol == :activitypub
-
-    case xml_root
-    when 'feed'
-      'Person'
-    when 'entry'
-      'Note'
-    end
   end
 
   def json_data
-    @_json_data ||= body_to_json(body)
-  end
-
-  def xml_root
-    xml_data.root.name
-  end
-
-  def xml_data
-    @_xml_data ||= Nokogiri::XML(body, nil, 'utf-8')
+    @json_data ||= body_to_json(body)
   end
 
   def local_url?
@@ -83,10 +68,10 @@ class ResolveURLService < BaseService
 
   def check_local_status(status)
     return if status.nil?
+
     authorize_with @on_behalf_of, status, :show?
     status
   rescue Mastodon::NotPermittedError
-    # Do not disclose the existence of status the user is not authorized to see
     nil
   end
 end
diff --git a/app/workers/activitypub/delivery_worker.rb b/app/workers/activitypub/delivery_worker.rb
index 8b52b8e49..5457d9d4b 100644
--- a/app/workers/activitypub/delivery_worker.rb
+++ b/app/workers/activitypub/delivery_worker.rb
@@ -2,6 +2,7 @@
 
 class ActivityPub::DeliveryWorker
   include Sidekiq::Worker
+  include JsonLdHelper
 
   STOPLIGHT_FAILURE_THRESHOLD = 10
   STOPLIGHT_COOLDOWN = 60
@@ -32,9 +33,10 @@ class ActivityPub::DeliveryWorker
   private
 
   def build_request(http_client)
-    request = Request.new(:post, @inbox_url, body: @json, http_client: http_client)
-    request.on_behalf_of(@source_account, :uri, sign_with: @options[:sign_with])
-    request.add_headers(HEADERS)
+    Request.new(:post, @inbox_url, body: @json, http_client: http_client).tap do |request|
+      request.on_behalf_of(@source_account, :uri, sign_with: @options[:sign_with])
+      request.add_headers(HEADERS)
+    end
   end
 
   def perform_request
@@ -53,14 +55,6 @@ class ActivityPub::DeliveryWorker
          .run
   end
 
-  def response_successful?(response)
-    (200...300).cover?(response.code)
-  end
-
-  def response_error_unsalvageable?(response)
-    response.code == 501 || ((400...500).cover?(response.code) && ![401, 408, 429].include?(response.code))
-  end
-
   def failure_tracker
     @failure_tracker ||= DeliveryFailureTracker.new(@inbox_url)
   end
diff --git a/spec/services/fetch_atom_service_spec.rb b/spec/services/fetch_atom_service_spec.rb
deleted file mode 100644
index 495540004..000000000
--- a/spec/services/fetch_atom_service_spec.rb
+++ /dev/null
@@ -1,96 +0,0 @@
-require 'rails_helper'
-
-RSpec.describe FetchAtomService, type: :service do
-  describe '#call' do
-    let(:url) { 'http://example.com' }
-    subject { FetchAtomService.new.call(url) }
-
-    context 'url is blank' do
-      let(:url) { '' }
-      it { is_expected.to be_nil }
-    end
-
-    context 'request failed' do
-      before do
-        WebMock.stub_request(:get, url).to_return(status: 500, body: '', headers: {})
-      end
-
-      it { is_expected.to be_nil }
-    end
-
-    context 'raise OpenSSL::SSL::SSLError' do
-      before do
-        allow(Request).to receive_message_chain(:new, :add_headers, :perform).and_raise(OpenSSL::SSL::SSLError)
-      end
-
-      it 'output log and return nil' do
-        expect_any_instance_of(ActiveSupport::Logger).to receive(:debug).with('SSL error: OpenSSL::SSL::SSLError')
-        is_expected.to be_nil
-      end
-    end
-
-    context 'raise HTTP::ConnectionError' do
-      before do
-        allow(Request).to receive_message_chain(:new, :add_headers, :perform).and_raise(HTTP::ConnectionError)
-      end
-
-      it 'output log and return nil' do
-        expect_any_instance_of(ActiveSupport::Logger).to receive(:debug).with('HTTP ConnectionError: HTTP::ConnectionError')
-        is_expected.to be_nil
-      end
-    end
-
-    context 'response success' do
-      let(:body) { '' }
-      let(:headers) { { 'Content-Type' => content_type } }
-      let(:json) {
-        { id: 1,
-          '@context': ActivityPub::TagManager::CONTEXT,
-          type: 'Note',
-        }.to_json
-      }
-
-      before do
-        WebMock.stub_request(:get, url).to_return(status: 200, body: body, headers: headers)
-      end
-
-      context 'content type is application/atom+xml' do
-        let(:content_type) { 'application/atom+xml' }
-
-        it { is_expected.to eq [url, { :prefetched_body => "" }, :ostatus] }
-      end
-
-      context 'content_type is activity+json' do
-        let(:content_type) { 'application/activity+json; charset=utf-8' }
-        let(:body) { json }
-
-        it { is_expected.to eq [1, { prefetched_body: body, id: true }, :activitypub] }
-      end
-
-      context 'content_type is ld+json with profile' do
-        let(:content_type) { 'application/ld+json; profile="https://www.w3.org/ns/activitystreams"' }
-        let(:body) { json }
-
-        it { is_expected.to eq [1, { prefetched_body: body, id: true }, :activitypub] }
-      end
-
-      before do
-        WebMock.stub_request(:get, url).to_return(status: 200, body: body, headers: headers)
-        WebMock.stub_request(:get, 'http://example.com/foo').to_return(status: 200, body: json, headers: { 'Content-Type' => 'application/activity+json' })
-      end
-
-      context 'has link header' do
-        let(:headers) { { 'Link' => '<http://example.com/foo>; rel="alternate"; type="application/activity+json"', } }
-
-        it { is_expected.to eq [1, { prefetched_body: json, id: true }, :activitypub] }
-      end
-
-      context 'content type is text/html' do
-        let(:content_type) { 'text/html' }
-        let(:body) { '<html><head><link rel="alternate" href="http://example.com/foo" type="application/activity+json"/></head></html>' }
-
-        it { is_expected.to eq [1, { prefetched_body: json, id: true }, :activitypub] }
-      end
-    end
-  end
-end
diff --git a/spec/services/fetch_remote_account_service_spec.rb b/spec/services/fetch_remote_account_service_spec.rb
index 37e9910d4..ee7325be2 100644
--- a/spec/services/fetch_remote_account_service_spec.rb
+++ b/spec/services/fetch_remote_account_service_spec.rb
@@ -4,6 +4,7 @@ RSpec.describe FetchRemoteAccountService, type: :service do
   let(:url) { 'https://example.com/alice' }
   let(:prefetched_body) { nil }
   let(:protocol) { :ostatus }
+
   subject { FetchRemoteAccountService.new.call(url, prefetched_body, protocol) }
 
   let(:actor) do
diff --git a/spec/services/fetch_resource_service_spec.rb b/spec/services/fetch_resource_service_spec.rb
new file mode 100644
index 000000000..17c192c44
--- /dev/null
+++ b/spec/services/fetch_resource_service_spec.rb
@@ -0,0 +1,96 @@
+require 'rails_helper'
+
+RSpec.describe FetchResourceService, type: :service do
+  let!(:representative) { Fabricate(:account) }
+
+  describe '#call' do
+    let(:url) { 'http://example.com' }
+    subject { described_class.new.call(url) }
+
+    context 'url is blank' do
+      let(:url) { '' }
+      it { is_expected.to be_nil }
+    end
+
+    context 'request failed' do
+      before do
+        WebMock.stub_request(:get, url).to_return(status: 500, body: '', headers: {})
+      end
+
+      it { is_expected.to be_nil }
+    end
+
+    context 'raise OpenSSL::SSL::SSLError' do
+      before do
+        allow(Request).to receive_message_chain(:new, :add_headers, :perform).and_raise(OpenSSL::SSL::SSLError)
+      end
+
+      it 'return nil' do
+        is_expected.to be_nil
+      end
+    end
+
+    context 'raise HTTP::ConnectionError' do
+      before do
+        allow(Request).to receive_message_chain(:new, :add_headers, :perform).and_raise(HTTP::ConnectionError)
+      end
+
+      it 'return nil' do
+        is_expected.to be_nil
+      end
+    end
+
+    context 'response success' do
+      let(:body) { '' }
+      let(:headers) { { 'Content-Type' => content_type } }
+      let(:json) {
+        { id: 1,
+          '@context': ActivityPub::TagManager::CONTEXT,
+          type: 'Note',
+        }.to_json
+      }
+
+      before do
+        WebMock.stub_request(:get, url).to_return(status: 200, body: body, headers: headers)
+      end
+
+      context 'content type is application/atom+xml' do
+        let(:content_type) { 'application/atom+xml' }
+
+        it { is_expected.to eq nil }
+      end
+
+      context 'content_type is activity+json' do
+        let(:content_type) { 'application/activity+json; charset=utf-8' }
+        let(:body) { json }
+
+        it { is_expected.to eq [1, { prefetched_body: body, id: true }, :activitypub] }
+      end
+
+      context 'content_type is ld+json with profile' do
+        let(:content_type) { 'application/ld+json; profile="https://www.w3.org/ns/activitystreams"' }
+        let(:body) { json }
+
+        it { is_expected.to eq [1, { prefetched_body: body, id: true }, :activitypub] }
+      end
+
+      before do
+        WebMock.stub_request(:get, url).to_return(status: 200, body: body, headers: headers)
+        WebMock.stub_request(:get, 'http://example.com/foo').to_return(status: 200, body: json, headers: { 'Content-Type' => 'application/activity+json' })
+      end
+
+      context 'has link header' do
+        let(:headers) { { 'Link' => '<http://example.com/foo>; rel="alternate"; type="application/activity+json"', } }
+
+        it { is_expected.to eq [1, { prefetched_body: json, id: true }, :activitypub] }
+      end
+
+      context 'content type is text/html' do
+        let(:content_type) { 'text/html' }
+        let(:body) { '<html><head><link rel="alternate" href="http://example.com/foo" type="application/activity+json"/></head></html>' }
+
+        it { is_expected.to eq [1, { prefetched_body: json, id: true }, :activitypub] }
+      end
+    end
+  end
+end
diff --git a/spec/services/resolve_url_service_spec.rb b/spec/services/resolve_url_service_spec.rb
index 7bb5d1940..aa4204637 100644
--- a/spec/services/resolve_url_service_spec.rb
+++ b/spec/services/resolve_url_service_spec.rb
@@ -6,48 +6,14 @@ describe ResolveURLService, type: :service do
   subject { described_class.new }
 
   describe '#call' do
-    it 'returns nil when there is no atom url' do
-      url = 'http://example.com/missing-atom'
+    it 'returns nil when there is no resource url' do
+      url     = 'http://example.com/missing-resource'
       service = double
-      allow(FetchAtomService).to receive(:new).and_return service
-      allow(service).to receive(:call).with(url).and_return(nil)
-
-      result = subject.call(url)
-      expect(result).to be_nil
-    end
-
-    it 'fetches remote accounts for feed types' do
-      url = 'http://example.com/atom-feed'
-      service = double
-      allow(FetchAtomService).to receive(:new).and_return service
-      feed_url = 'http://feed-url'
-      feed_content = '<feed>contents</feed>'
-      allow(service).to receive(:call).with(url).and_return([feed_url, { prefetched_body: feed_content }])
-
-      account_service = double
-      allow(FetchRemoteAccountService).to receive(:new).and_return(account_service)
-      allow(account_service).to receive(:call)
-
-      _result = subject.call(url)
 
-      expect(account_service).to have_received(:call).with(feed_url, feed_content, nil)
-    end
-
-    it 'fetches remote statuses for entry types' do
-      url = 'http://example.com/atom-entry'
-      service = double
-      allow(FetchAtomService).to receive(:new).and_return service
-      feed_url = 'http://feed-url'
-      feed_content = '<entry>contents</entry>'
-      allow(service).to receive(:call).with(url).and_return([feed_url, { prefetched_body: feed_content }])
-
-      account_service = double
-      allow(FetchRemoteStatusService).to receive(:new).and_return(account_service)
-      allow(account_service).to receive(:call)
-
-      _result = subject.call(url)
+      allow(FetchResourceService).to receive(:new).and_return service
+      allow(service).to receive(:call).with(url).and_return(nil)
 
-      expect(account_service).to have_received(:call).with(feed_url, feed_content, nil)
+      expect(subject.call(url)).to be_nil
     end
   end
 end
-- 
cgit 


From 4e8dcc5dbbf625b7268ed10d36122de985da6bdc Mon Sep 17 00:00:00 2001
From: Eugen Rochko <eugen@zeonfederated.com>
Date: Thu, 11 Jul 2019 14:49:55 +0200
Subject: Add HTTP signatures to all outgoing ActivityPub GET requests (#11284)

---
 app/helpers/jsonld_helper.rb                       | 13 ++---
 app/lib/request.rb                                 |  4 +-
 app/services/fetch_resource_service.rb             |  2 +-
 .../concerns/signature_verification_spec.rb        |  2 +-
 spec/services/fetch_remote_account_service_spec.rb |  1 +
 spec/services/fetch_resource_service_spec.rb       | 61 +++++++++++++---------
 6 files changed, 43 insertions(+), 40 deletions(-)

(limited to 'spec')

diff --git a/app/helpers/jsonld_helper.rb b/app/helpers/jsonld_helper.rb
index 34a657e06..83a5b2462 100644
--- a/app/helpers/jsonld_helper.rb
+++ b/app/helpers/jsonld_helper.rb
@@ -77,19 +77,12 @@ module JsonLdHelper
   end
 
   def fetch_resource_without_id_validation(uri, on_behalf_of = nil, raise_on_temporary_error = false)
-    build_request(uri, on_behalf_of).perform do |response|
-      raise Mastodon::UnexpectedResponseError, response unless response_successful?(response) || response_error_unsalvageable?(response) || !raise_on_temporary_error
-
-      return body_to_json(response.body_with_limit) if response.code == 200
-    end
-
-    # If request failed, retry without doing it on behalf of a user
-    return if on_behalf_of.nil?
+    on_behalf_of ||= Account.representative
 
-    build_request(uri).perform do |response|
+    build_request(uri, on_behalf_of).perform do |response|
       raise Mastodon::UnexpectedResponseError, response unless response_successful?(response) || response_error_unsalvageable?(response) || !raise_on_temporary_error
 
-      response.code == 200 ? body_to_json(response.body_with_limit) : nil
+      body_to_json(response.body_with_limit) if response.code == 200
     end
   end
 
diff --git a/app/lib/request.rb b/app/lib/request.rb
index 1fd3f5190..9d874fe2c 100644
--- a/app/lib/request.rb
+++ b/app/lib/request.rb
@@ -40,8 +40,8 @@ class Request
     set_digest! if options.key?(:body)
   end
 
-  def on_behalf_of(account, key_id_format = :acct, sign_with: nil)
-    raise ArgumentError, 'account must be local' unless account&.local?
+  def on_behalf_of(account, key_id_format = :uri, sign_with: nil)
+    raise ArgumentError, 'account must not be nil' if account.nil?
 
     @account       = account
     @keypair       = sign_with.present? ? OpenSSL::PKey::RSA.new(sign_with) : @account.keypair
diff --git a/app/services/fetch_resource_service.rb b/app/services/fetch_resource_service.rb
index c0473f3ad..3676d899d 100644
--- a/app/services/fetch_resource_service.rb
+++ b/app/services/fetch_resource_service.rb
@@ -23,7 +23,7 @@ class FetchResourceService < BaseService
   end
 
   def perform_request(&block)
-    Request.new(:get, @url).add_headers('Accept' => ACCEPT_HEADER).perform(&block)
+    Request.new(:get, @url).add_headers('Accept' => ACCEPT_HEADER).on_behalf_of(Account.representative).perform(&block)
   end
 
   def process_response(response, terminal = false)
diff --git a/spec/controllers/concerns/signature_verification_spec.rb b/spec/controllers/concerns/signature_verification_spec.rb
index 720690097..1fa19f54d 100644
--- a/spec/controllers/concerns/signature_verification_spec.rb
+++ b/spec/controllers/concerns/signature_verification_spec.rb
@@ -38,7 +38,7 @@ describe ApplicationController, type: :controller do
   end
 
   context 'with signature header' do
-    let!(:author) { Fabricate(:account) }
+    let!(:author) { Fabricate(:account, domain: 'example.com', uri: 'https://example.com/actor') }
 
     context 'without body' do
       before do
diff --git a/spec/services/fetch_remote_account_service_spec.rb b/spec/services/fetch_remote_account_service_spec.rb
index ee7325be2..b37445861 100644
--- a/spec/services/fetch_remote_account_service_spec.rb
+++ b/spec/services/fetch_remote_account_service_spec.rb
@@ -4,6 +4,7 @@ RSpec.describe FetchRemoteAccountService, type: :service do
   let(:url) { 'https://example.com/alice' }
   let(:prefetched_body) { nil }
   let(:protocol) { :ostatus }
+  let!(:representative) { Fabricate(:account) }
 
   subject { FetchRemoteAccountService.new.call(url, prefetched_body, protocol) }
 
diff --git a/spec/services/fetch_resource_service_spec.rb b/spec/services/fetch_resource_service_spec.rb
index 17c192c44..98630966b 100644
--- a/spec/services/fetch_resource_service_spec.rb
+++ b/spec/services/fetch_resource_service_spec.rb
@@ -5,69 +5,78 @@ RSpec.describe FetchResourceService, type: :service do
 
   describe '#call' do
     let(:url) { 'http://example.com' }
+
     subject { described_class.new.call(url) }
 
-    context 'url is blank' do
+    context 'with blank url' do
       let(:url) { '' }
       it { is_expected.to be_nil }
     end
 
-    context 'request failed' do
+    context 'when request fails' do
       before do
-        WebMock.stub_request(:get, url).to_return(status: 500, body: '', headers: {})
+        stub_request(:get, url).to_return(status: 500, body: '', headers: {})
       end
 
       it { is_expected.to be_nil }
     end
 
-    context 'raise OpenSSL::SSL::SSLError' do
+    context 'when OpenSSL::SSL::SSLError is raised' do
       before do
-        allow(Request).to receive_message_chain(:new, :add_headers, :perform).and_raise(OpenSSL::SSL::SSLError)
+        allow(Request).to receive_message_chain(:new, :add_headers, :on_behalf_of, :perform).and_raise(OpenSSL::SSL::SSLError)
       end
 
-      it 'return nil' do
-        is_expected.to be_nil
-      end
+      it { is_expected.to be_nil }
     end
 
-    context 'raise HTTP::ConnectionError' do
+    context 'when HTTP::ConnectionError is raised' do
       before do
-        allow(Request).to receive_message_chain(:new, :add_headers, :perform).and_raise(HTTP::ConnectionError)
+        allow(Request).to receive_message_chain(:new, :add_headers, :on_behalf_of, :perform).and_raise(HTTP::ConnectionError)
       end
 
-      it 'return nil' do
-        is_expected.to be_nil
-      end
+      it { is_expected.to be_nil }
     end
 
-    context 'response success' do
+    context 'when request succeeds' do
       let(:body) { '' }
-      let(:headers) { { 'Content-Type' => content_type } }
-      let(:json) {
-        { id: 1,
+
+      let(:content_type) { 'application/json' }
+
+      let(:headers) do
+        { 'Content-Type' => content_type }
+      end
+
+      let(:json) do
+        {
+          id: 1,
           '@context': ActivityPub::TagManager::CONTEXT,
           type: 'Note',
         }.to_json
-      }
+      end
 
       before do
-        WebMock.stub_request(:get, url).to_return(status: 200, body: body, headers: headers)
+        stub_request(:get, url).to_return(status: 200, body: body, headers: headers)
+      end
+
+      it 'signs request' do
+        subject
+        expect(a_request(:get, url).with(headers: { 'Signature' => /keyId="#{Regexp.escape(ActivityPub::TagManager.instance.uri_for(representative) + '#main-key')}"/ })).to have_been_made
       end
 
-      context 'content type is application/atom+xml' do
+      context 'when content type is application/atom+xml' do
         let(:content_type) { 'application/atom+xml' }
 
         it { is_expected.to eq nil }
       end
 
-      context 'content_type is activity+json' do
+      context 'when content type is activity+json' do
         let(:content_type) { 'application/activity+json; charset=utf-8' }
         let(:body) { json }
 
         it { is_expected.to eq [1, { prefetched_body: body, id: true }, :activitypub] }
       end
 
-      context 'content_type is ld+json with profile' do
+      context 'when content type is ld+json with profile' do
         let(:content_type) { 'application/ld+json; profile="https://www.w3.org/ns/activitystreams"' }
         let(:body) { json }
 
@@ -75,17 +84,17 @@ RSpec.describe FetchResourceService, type: :service do
       end
 
       before do
-        WebMock.stub_request(:get, url).to_return(status: 200, body: body, headers: headers)
-        WebMock.stub_request(:get, 'http://example.com/foo').to_return(status: 200, body: json, headers: { 'Content-Type' => 'application/activity+json' })
+        stub_request(:get, url).to_return(status: 200, body: body, headers: headers)
+        stub_request(:get, 'http://example.com/foo').to_return(status: 200, body: json, headers: { 'Content-Type' => 'application/activity+json' })
       end
 
-      context 'has link header' do
+      context 'when link header is present' do
         let(:headers) { { 'Link' => '<http://example.com/foo>; rel="alternate"; type="application/activity+json"', } }
 
         it { is_expected.to eq [1, { prefetched_body: json, id: true }, :activitypub] }
       end
 
-      context 'content type is text/html' do
+      context 'when content type is text/html' do
         let(:content_type) { 'text/html' }
         let(:body) { '<html><head><link rel="alternate" href="http://example.com/foo" type="application/activity+json"/></head></html>' }
 
-- 
cgit 


From 5bf67ca91350e40e6f329271d3ca2bdcba87ab64 Mon Sep 17 00:00:00 2001
From: Eugen Rochko <eugen@zeonfederated.com>
Date: Thu, 11 Jul 2019 20:11:09 +0200
Subject: Add ActivityPub secure mode (#11269)

* Add HTTP signature requirement for served ActivityPub resources

* Change `SECURE_MODE` to `AUTHORIZED_FETCH`

* Add 'Signature' to 'Vary' header and improve code style

* Improve code style by adding `public_fetch_mode?` method
---
 app/controllers/accounts_controller.rb             | 13 +++++++++--
 .../activitypub/collections_controller.rb          |  3 ++-
 app/controllers/activitypub/inboxes_controller.rb  | 27 +++++++++++++---------
 app/controllers/activitypub/outboxes_controller.rb |  4 ++--
 app/controllers/activitypub/replies_controller.rb  |  2 ++
 app/controllers/application_controller.rb          | 10 +++++++-
 .../concerns/account_controller_concern.rb         |  2 +-
 app/controllers/concerns/signature_verification.rb | 19 ++++++++++++---
 app/controllers/follower_accounts_controller.rb    | 12 +++++++---
 app/controllers/following_accounts_controller.rb   | 12 +++++++---
 app/controllers/statuses_controller.rb             |  9 ++++----
 app/controllers/tags_controller.rb                 |  5 +++-
 app/lib/activitypub/adapter.rb                     |  1 +
 .../activitypub/inboxes_controller_spec.rb         |  4 ++--
 14 files changed, 89 insertions(+), 34 deletions(-)

(limited to 'spec')

diff --git a/app/controllers/accounts_controller.rb b/app/controllers/accounts_controller.rb
index 3184a73cb..fc913c2ec 100644
--- a/app/controllers/accounts_controller.rb
+++ b/app/controllers/accounts_controller.rb
@@ -4,6 +4,7 @@ class AccountsController < ApplicationController
   PAGE_SIZE = 20
 
   include AccountControllerConcern
+  include SignatureAuthentication
 
   before_action :set_cache_headers
   before_action :set_body_classes
@@ -39,8 +40,8 @@ class AccountsController < ApplicationController
       end
 
       format.json do
-        expires_in 3.minutes, public: true
-        render json: @account, content_type: 'application/activity+json', serializer: ActivityPub::ActorSerializer, adapter: ActivityPub::Adapter
+        expires_in 3.minutes, public: !(authorized_fetch_mode? && signed_request_account.present?)
+        render json: @account, content_type: 'application/activity+json', serializer: ActivityPub::ActorSerializer, adapter: ActivityPub::Adapter, fields: restrict_fields_to
       end
     end
   end
@@ -132,4 +133,12 @@ class AccountsController < ApplicationController
       filtered_statuses.paginate_by_max_id(PAGE_SIZE, params[:max_id], params[:since_id]).to_a
     end
   end
+
+  def restrict_fields_to
+    if signed_request_account.present? || public_fetch_mode?
+      # Return all fields
+    else
+      %i(id type preferred_username inbox public_key endpoints)
+    end
+  end
 end
diff --git a/app/controllers/activitypub/collections_controller.rb b/app/controllers/activitypub/collections_controller.rb
index dd2f111b0..035467f41 100644
--- a/app/controllers/activitypub/collections_controller.rb
+++ b/app/controllers/activitypub/collections_controller.rb
@@ -4,12 +4,13 @@ class ActivityPub::CollectionsController < Api::BaseController
   include SignatureVerification
   include AccountOwnedConcern
 
+  before_action :require_signature!, if: :authorized_fetch_mode?
   before_action :set_size
   before_action :set_statuses
   before_action :set_cache_headers
 
   def show
-    expires_in 3.minutes, public: true
+    expires_in 3.minutes, public: public_fetch_mode?
     render json: collection_presenter, content_type: 'application/activity+json', serializer: ActivityPub::CollectionSerializer, adapter: ActivityPub::Adapter, skip_activities: true
   end
 
diff --git a/app/controllers/activitypub/inboxes_controller.rb b/app/controllers/activitypub/inboxes_controller.rb
index 9be0676e1..7cfd9a25e 100644
--- a/app/controllers/activitypub/inboxes_controller.rb
+++ b/app/controllers/activitypub/inboxes_controller.rb
@@ -5,23 +5,24 @@ class ActivityPub::InboxesController < Api::BaseController
   include JsonLdHelper
   include AccountOwnedConcern
 
+  before_action :skip_unknown_actor_delete
+  before_action :require_signature!
+
   def create
-    if unknown_deleted_account?
-      head 202
-    elsif signed_request_account
-      upgrade_account
-      process_payload
-      head 202
-    else
-      render plain: signature_verification_failure_reason, status: 401
-    end
+    upgrade_account
+    process_payload
+    head 202
   end
 
   private
 
+  def skip_unknown_actor_delete
+    head 202 if unknown_deleted_account?
+  end
+
   def unknown_deleted_account?
     json = Oj.load(body, mode: :strict)
-    json['type'] == 'Delete' && json['actor'].present? && json['actor'] == value_or_id(json['object']) && !Account.where(uri: json['actor']).exists?
+    json.is_a?(Hash) && json['type'] == 'Delete' && json['actor'].present? && json['actor'] == value_or_id(json['object']) && !Account.where(uri: json['actor']).exists?
   rescue Oj::ParseError
     false
   end
@@ -32,8 +33,12 @@ class ActivityPub::InboxesController < Api::BaseController
 
   def body
     return @body if defined?(@body)
-    @body = request.body.read.force_encoding('UTF-8')
+
+    @body = request.body.read
+    @body.force_encoding('UTF-8') if @body.present?
+
     request.body.rewind if request.body.respond_to?(:rewind)
+
     @body
   end
 
diff --git a/app/controllers/activitypub/outboxes_controller.rb b/app/controllers/activitypub/outboxes_controller.rb
index 4c0b769f0..cdfd28ba8 100644
--- a/app/controllers/activitypub/outboxes_controller.rb
+++ b/app/controllers/activitypub/outboxes_controller.rb
@@ -6,12 +6,12 @@ class ActivityPub::OutboxesController < Api::BaseController
   include SignatureVerification
   include AccountOwnedConcern
 
+  before_action :require_signature!, if: :authorized_fetch_mode?
   before_action :set_statuses
   before_action :set_cache_headers
 
   def show
-    expires_in 1.minute, public: true unless page_requested?
-
+    expires_in(page_requested? ? 0 : 3.minutes, public: public_fetch_mode?)
     render json: outbox_presenter, serializer: ActivityPub::OutboxSerializer, adapter: ActivityPub::Adapter, content_type: 'application/activity+json'
   end
 
diff --git a/app/controllers/activitypub/replies_controller.rb b/app/controllers/activitypub/replies_controller.rb
index 99b7b310f..020c077ab 100644
--- a/app/controllers/activitypub/replies_controller.rb
+++ b/app/controllers/activitypub/replies_controller.rb
@@ -7,11 +7,13 @@ class ActivityPub::RepliesController < Api::BaseController
 
   DESCENDANTS_LIMIT = 60
 
+  before_action :require_signature!, if: :authorized_fetch_mode?
   before_action :set_status
   before_action :set_cache_headers
   before_action :set_replies
 
   def index
+    expires_in 0, public: public_fetch_mode?
     render json: replies_collection_presenter, serializer: ActivityPub::CollectionSerializer, adapter: ActivityPub::Adapter, content_type: 'application/activity+json', skip_activities: true
   end
 
diff --git a/app/controllers/application_controller.rb b/app/controllers/application_controller.rb
index cc8b8e4da..16e7d70a3 100644
--- a/app/controllers/application_controller.rb
+++ b/app/controllers/application_controller.rb
@@ -36,6 +36,14 @@ class ApplicationController < ActionController::Base
     Rails.env.production?
   end
 
+  def authorized_fetch_mode?
+    ENV['AUTHORIZED_FETCH'] == 'true'
+  end
+
+  def public_fetch_mode?
+    !authorized_fetch_mode?
+  end
+
   def store_current_location
     store_location_for(:user, request.url) unless request.format == :json
   end
@@ -152,6 +160,6 @@ class ApplicationController < ActionController::Base
   end
 
   def set_cache_headers
-    response.headers['Vary'] = 'Accept'
+    response.headers['Vary'] = 'Accept, Signature'
   end
 end
diff --git a/app/controllers/concerns/account_controller_concern.rb b/app/controllers/concerns/account_controller_concern.rb
index 287a930da..11eac0eb6 100644
--- a/app/controllers/concerns/account_controller_concern.rb
+++ b/app/controllers/concerns/account_controller_concern.rb
@@ -11,7 +11,7 @@ module AccountControllerConcern
     layout 'public'
 
     before_action :set_instance_presenter
-    before_action :set_link_headers
+    before_action :set_link_headers, if: -> { request.format.nil? || request.format == :html }
   end
 
   private
diff --git a/app/controllers/concerns/signature_verification.rb b/app/controllers/concerns/signature_verification.rb
index 0ccdf5ec9..7b251cf80 100644
--- a/app/controllers/concerns/signature_verification.rb
+++ b/app/controllers/concerns/signature_verification.rb
@@ -7,12 +7,20 @@ module SignatureVerification
 
   include DomainControlHelper
 
+  def require_signature!
+    render plain: signature_verification_failure_reason, status: signature_verification_failure_code unless signed_request_account
+  end
+
   def signed_request?
     request.headers['Signature'].present?
   end
 
   def signature_verification_failure_reason
-    return @signature_verification_failure_reason if defined?(@signature_verification_failure_reason)
+    @signature_verification_failure_reason
+  end
+
+  def signature_verification_failure_code
+    @signature_verification_failure_code || 401
   end
 
   def signed_request_account
@@ -125,11 +133,16 @@ module SignatureVerification
   end
 
   def account_from_key_id(key_id)
+    domain = key_id.start_with?('acct:') ? key_id.split('@').last : key_id
+
+    if domain_not_allowed?(domain)
+      @signature_verification_failure_code = 403
+      return
+    end
+
     if key_id.start_with?('acct:')
       stoplight_wrap_request { ResolveAccountService.new.call(key_id.gsub(/\Aacct:/, '')) }
     elsif !ActivityPub::TagManager.instance.local_uri?(key_id)
-      return if domain_not_allowed?(key_id)
-
       account   = ActivityPub::TagManager.instance.uri_to_resource(key_id, Account)
       account ||= stoplight_wrap_request { ActivityPub::FetchRemoteKeyService.new.call(key_id, id: false) }
       account
diff --git a/app/controllers/follower_accounts_controller.rb b/app/controllers/follower_accounts_controller.rb
index 8baa64490..6e873de5b 100644
--- a/app/controllers/follower_accounts_controller.rb
+++ b/app/controllers/follower_accounts_controller.rb
@@ -2,7 +2,9 @@
 
 class FollowerAccountsController < ApplicationController
   include AccountControllerConcern
+  include SignatureVerification
 
+  before_action :require_signature!, if: -> { request.format == :json && authorized_fetch_mode? }
   before_action :set_cache_headers
 
   def index
@@ -17,9 +19,9 @@ class FollowerAccountsController < ApplicationController
       end
 
       format.json do
-        raise Mastodon::NotPermittedError if params[:page].present? && @account.user_hides_network?
+        raise Mastodon::NotPermittedError if page_requested? && @account.user_hides_network?
 
-        expires_in 3.minutes, public: true if params[:page].blank?
+        expires_in(page_requested? ? 0 : 3.minutes, public: public_fetch_mode?)
 
         render json: collection_presenter,
                serializer: ActivityPub::CollectionSerializer,
@@ -35,12 +37,16 @@ class FollowerAccountsController < ApplicationController
     @follows ||= Follow.where(target_account: @account).recent.page(params[:page]).per(FOLLOW_PER_PAGE).preload(:account)
   end
 
+  def page_requested?
+    params[:page].present?
+  end
+
   def page_url(page)
     account_followers_url(@account, page: page) unless page.nil?
   end
 
   def collection_presenter
-    if params[:page].present?
+    if page_requested?
       ActivityPub::CollectionPresenter.new(
         id: account_followers_url(@account, page: params.fetch(:page, 1)),
         type: :ordered,
diff --git a/app/controllers/following_accounts_controller.rb b/app/controllers/following_accounts_controller.rb
index 4d1ea4594..07d62f7dd 100644
--- a/app/controllers/following_accounts_controller.rb
+++ b/app/controllers/following_accounts_controller.rb
@@ -2,7 +2,9 @@
 
 class FollowingAccountsController < ApplicationController
   include AccountControllerConcern
+  include SignatureVerification
 
+  before_action :require_signature!, if: -> { request.format == :json && authorized_fetch_mode? }
   before_action :set_cache_headers
 
   def index
@@ -17,9 +19,9 @@ class FollowingAccountsController < ApplicationController
       end
 
       format.json do
-        raise Mastodon::NotPermittedError if params[:page].present? && @account.user_hides_network?
+        raise Mastodon::NotPermittedError if page_requested? && @account.user_hides_network?
 
-        expires_in 3.minutes, public: true if params[:page].blank?
+        expires_in(page_requested? ? 0 : 3.minutes, public: public_fetch_mode?)
 
         render json: collection_presenter,
                serializer: ActivityPub::CollectionSerializer,
@@ -35,12 +37,16 @@ class FollowingAccountsController < ApplicationController
     @follows ||= Follow.where(account: @account).recent.page(params[:page]).per(FOLLOW_PER_PAGE).preload(:target_account)
   end
 
+  def page_requested?
+    params[:page].present?
+  end
+
   def page_url(page)
     account_following_index_url(@account, page: page) unless page.nil?
   end
 
   def collection_presenter
-    if params[:page].present?
+    if page_requested?
       ActivityPub::CollectionPresenter.new(
         id: account_following_index_url(@account, page: params.fetch(:page, 1)),
         type: :ordered,
diff --git a/app/controllers/statuses_controller.rb b/app/controllers/statuses_controller.rb
index 13ce5c691..22e7519f9 100644
--- a/app/controllers/statuses_controller.rb
+++ b/app/controllers/statuses_controller.rb
@@ -8,11 +8,12 @@ class StatusesController < ApplicationController
 
   layout 'public'
 
+  before_action :require_signature!, only: :show, if: -> { request.format == :json && authorized_fetch_mode? }
   before_action :set_status
   before_action :set_instance_presenter
   before_action :set_link_headers
-  before_action :redirect_to_original, only: [:show]
-  before_action :set_referrer_policy_header, only: [:show]
+  before_action :redirect_to_original, only: :show
+  before_action :set_referrer_policy_header, only: :show
   before_action :set_cache_headers
   before_action :set_body_classes
   before_action :set_autoplay, only: :embed
@@ -30,14 +31,14 @@ class StatusesController < ApplicationController
       end
 
       format.json do
-        expires_in 3.minutes, public: @status.distributable?
+        expires_in 3.minutes, public: @status.distributable? && public_fetch_mode?
         render json: @status, content_type: 'application/activity+json', serializer: ActivityPub::NoteSerializer, adapter: ActivityPub::Adapter
       end
     end
   end
 
   def activity
-    expires_in 3.minutes, public: @status.distributable?
+    expires_in 3.minutes, public: @status.distributable? && public_fetch_mode?
     render json: @status, content_type: 'application/activity+json', serializer: ActivityPub::ActivitySerializer, adapter: ActivityPub::Adapter
   end
 
diff --git a/app/controllers/tags_controller.rb b/app/controllers/tags_controller.rb
index 2ecce0ca2..d08e5a61a 100644
--- a/app/controllers/tags_controller.rb
+++ b/app/controllers/tags_controller.rb
@@ -1,10 +1,13 @@
 # frozen_string_literal: true
 
 class TagsController < ApplicationController
+  include SignatureVerification
+
   PAGE_SIZE = 20
 
   layout 'public'
 
+  before_action :require_signature!, if: -> { request.format == :json && authorized_fetch_mode? }
   before_action :set_tag
   before_action :set_body_classes
   before_action :set_instance_presenter
@@ -30,7 +33,7 @@ class TagsController < ApplicationController
       end
 
       format.json do
-        expires_in 3.minutes, public: true
+        expires_in 3.minutes, public: public_fetch_mode?
 
         @statuses = HashtagQueryService.new.call(@tag, params.slice(:any, :all, :none), current_account, params[:local]).paginate_by_max_id(PAGE_SIZE, params[:max_id])
         @statuses = cache_collection(@statuses, Status)
diff --git a/app/lib/activitypub/adapter.rb b/app/lib/activitypub/adapter.rb
index c259c96f4..a1d84de2f 100644
--- a/app/lib/activitypub/adapter.rb
+++ b/app/lib/activitypub/adapter.rb
@@ -33,6 +33,7 @@ class ActivityPub::Adapter < ActiveModelSerializers::Adapter::Base
   def serializable_hash(options = nil)
     options         = serialization_options(options)
     serialized_hash = serializer.serializable_hash(options)
+    serialized_hash = serialized_hash.select { |k, _| options[:fields].include?(k) } if options[:fields]
     serialized_hash = self.class.transform_key_casing!(serialized_hash, instance_options)
 
     { '@context' => serialized_context }.merge(serialized_hash)
diff --git a/spec/controllers/activitypub/inboxes_controller_spec.rb b/spec/controllers/activitypub/inboxes_controller_spec.rb
index eab4b8c3e..a9ee75490 100644
--- a/spec/controllers/activitypub/inboxes_controller_spec.rb
+++ b/spec/controllers/activitypub/inboxes_controller_spec.rb
@@ -4,7 +4,7 @@ require 'rails_helper'
 
 RSpec.describe ActivityPub::InboxesController, type: :controller do
   describe 'POST #create' do
-    context 'if signed_request_account' do
+    context 'with signed_request_account' do
       it 'returns 202' do
         allow(controller).to receive(:signed_request_account) do
           Fabricate(:account)
@@ -15,7 +15,7 @@ RSpec.describe ActivityPub::InboxesController, type: :controller do
       end
     end
 
-    context 'not signed_request_account' do
+    context 'without signed_request_account' do
       it 'returns 401' do
         allow(controller).to receive(:signed_request_account) do
           false
-- 
cgit 


From 6ff67be0f6e79ec403e08c69717ee8c89451c70e Mon Sep 17 00:00:00 2001
From: Eugen Rochko <eugen@zeonfederated.com>
Date: Sat, 13 Jul 2019 16:45:50 +0200
Subject: Add a spam check (#11217)

* Add a spam check

* Use Nilsimsa to generate locality-sensitive hashes and compare using Levenshtein distance

* Add more tests

* Add exemption when the message is a reply to something that mentions the sender

* Use Nilsimsa Compare Value instead of Levenshtein distance

* Use MD5 for messages shorter than 10 characters

* Add message to automated report, do not add non-public statuses to
automated report, add trust level to accounts and make unsilencing
raise the trust level to prevent repeated spam checks on that account

* Expire spam check data after 3 months

* Add support for local statuses, reduce expiration to 1 week, always create a report

* Add content warnings to the spam check and exempt empty statuses

* Change Nilsimsa threshold to 95 and make sure removed statuses are removed from the spam check

* Add all matched statuses into automatic report
---
 Gemfile                                            |   1 +
 Gemfile.lock                                       |   8 +
 app/lib/activitypub/activity/create.rb             |  13 ++
 app/lib/spam_check.rb                              | 169 +++++++++++++++++++++
 app/models/account.rb                              |  18 ++-
 app/services/remove_status_service.rb              |   5 +
 config/locales/en.yml                              |   2 +
 .../20190701022101_add_trust_level_to_accounts.rb  |   5 +
 db/schema.rb                                       |   1 +
 spec/lib/spam_check_spec.rb                        | 160 +++++++++++++++++++
 10 files changed, 377 insertions(+), 5 deletions(-)
 create mode 100644 app/lib/spam_check.rb
 create mode 100644 db/migrate/20190701022101_add_trust_level_to_accounts.rb
 create mode 100644 spec/lib/spam_check_spec.rb

(limited to 'spec')

diff --git a/Gemfile b/Gemfile
index 613515628..15334678b 100644
--- a/Gemfile
+++ b/Gemfile
@@ -58,6 +58,7 @@ gem 'idn-ruby', require: 'idn'
 gem 'kaminari', '~> 1.1'
 gem 'link_header', '~> 0.0'
 gem 'mime-types', '~> 3.2', require: 'mime/types/columnar'
+gem 'nilsimsa', git: 'https://github.com/witgo/nilsimsa', ref: 'fd184883048b922b176939f851338d0a4971a532'
 gem 'nokogiri', '~> 1.10'
 gem 'nsa', '~> 0.2'
 gem 'oj', '~> 3.7'
diff --git a/Gemfile.lock b/Gemfile.lock
index 340bbcdd8..c3198b7d9 100644
--- a/Gemfile.lock
+++ b/Gemfile.lock
@@ -12,6 +12,13 @@ GIT
   specs:
     http_parser.rb (0.6.1)
 
+GIT
+  remote: https://github.com/witgo/nilsimsa
+  revision: fd184883048b922b176939f851338d0a4971a532
+  ref: fd184883048b922b176939f851338d0a4971a532
+  specs:
+    nilsimsa (1.1.2)
+
 GEM
   remote: https://rubygems.org/
   specs:
@@ -704,6 +711,7 @@ DEPENDENCIES
   microformats (~> 4.1)
   mime-types (~> 3.2)
   net-ldap (~> 0.10)
+  nilsimsa!
   nokogiri (~> 1.10)
   nsa (~> 0.2)
   oj (~> 3.7)
diff --git a/app/lib/activitypub/activity/create.rb b/app/lib/activitypub/activity/create.rb
index 5849c20d7..56c24680a 100644
--- a/app/lib/activitypub/activity/create.rb
+++ b/app/lib/activitypub/activity/create.rb
@@ -41,6 +41,7 @@ class ActivityPub::Activity::Create < ActivityPub::Activity
 
     resolve_thread(@status)
     fetch_replies(@status)
+    check_for_spam
     distribute(@status)
     forward_for_reply if @status.distributable?
   end
@@ -406,6 +407,18 @@ class ActivityPub::Activity::Create < ActivityPub::Activity
     Account.local.where(username: local_usernames).exists?
   end
 
+  def check_for_spam
+    spam_check = SpamCheck.new(@status)
+
+    return if spam_check.skip?
+
+    if spam_check.spam?
+      spam_check.flag!
+    else
+      spam_check.remember!
+    end
+  end
+
   def forward_for_reply
     return unless @json['signature'].present? && reply_to_local?
     ActivityPub::RawDistributionWorker.perform_async(Oj.dump(@json), replied_to_status.account_id, [@account.preferred_inbox_url])
diff --git a/app/lib/spam_check.rb b/app/lib/spam_check.rb
new file mode 100644
index 000000000..923d48a02
--- /dev/null
+++ b/app/lib/spam_check.rb
@@ -0,0 +1,169 @@
+# frozen_string_literal: true
+
+class SpamCheck
+  include Redisable
+  include ActionView::Helpers::TextHelper
+
+  NILSIMSA_COMPARE_THRESHOLD = 95
+  NILSIMSA_MIN_SIZE          = 10
+  EXPIRE_SET_AFTER           = 1.week.seconds
+
+  def initialize(status)
+    @account = status.account
+    @status  = status
+  end
+
+  def skip?
+    already_flagged? || trusted? || no_unsolicited_mentions? || solicited_reply?
+  end
+
+  def spam?
+    if insufficient_data?
+      false
+    elsif nilsimsa?
+      any_other_digest?('nilsimsa') { |_, other_digest| nilsimsa_compare_value(digest, other_digest) >= NILSIMSA_COMPARE_THRESHOLD }
+    else
+      any_other_digest?('md5') { |_, other_digest| other_digest == digest }
+    end
+  end
+
+  def flag!
+    auto_silence_account!
+    auto_report_status!
+  end
+
+  def remember!
+    # The scores in sorted sets don't actually have enough bits to hold an exact
+    # value of our snowflake IDs, so we use it only for its ordering property. To
+    # get the correct status ID back, we have to save it in the string value
+
+    redis.zadd(redis_key, @status.id, digest_with_algorithm)
+    redis.zremrangebyrank(redis_key, '0', '-10')
+    redis.expire(redis_key, EXPIRE_SET_AFTER)
+  end
+
+  def reset!
+    redis.del(redis_key)
+  end
+
+  def hashable_text
+    return @hashable_text if defined?(@hashable_text)
+
+    @hashable_text = @status.text
+    @hashable_text = remove_mentions(@hashable_text)
+    @hashable_text = strip_tags(@hashable_text) unless @status.local?
+    @hashable_text = normalize_unicode(@status.spoiler_text + ' ' + @hashable_text)
+    @hashable_text = remove_whitespace(@hashable_text)
+  end
+
+  def insufficient_data?
+    hashable_text.blank?
+  end
+
+  def digest
+    @digest ||= begin
+      if nilsimsa?
+        Nilsimsa.new(hashable_text).hexdigest
+      else
+        Digest::MD5.hexdigest(hashable_text)
+      end
+    end
+  end
+
+  def digest_with_algorithm
+    if nilsimsa?
+      ['nilsimsa', digest, @status.id].join(':')
+    else
+      ['md5', digest, @status.id].join(':')
+    end
+  end
+
+  private
+
+  def remove_mentions(text)
+    return text.gsub(Account::MENTION_RE, '') if @status.local?
+
+    Nokogiri::HTML.fragment(text).tap do |html|
+      mentions = @status.mentions.map { |mention| ActivityPub::TagManager.instance.url_for(mention.account) }
+
+      html.traverse do |element|
+        element.unlink if element.name == 'a' && mentions.include?(element['href'])
+      end
+    end.to_s
+  end
+
+  def normalize_unicode(text)
+    text.unicode_normalize(:nfkc).downcase
+  end
+
+  def remove_whitespace(text)
+    text.gsub(/\s+/, ' ').strip
+  end
+
+  def auto_silence_account!
+    @account.silence!
+  end
+
+  def auto_report_status!
+    status_ids = Status.where(visibility: %i(public unlisted)).where(id: matching_status_ids).pluck(:id) + [@status.id] if @status.distributable?
+    ReportService.new.call(Account.representative, @account, status_ids: status_ids, comment: I18n.t('spam_check.spam_detected_and_silenced'))
+  end
+
+  def already_flagged?
+    @account.silenced?
+  end
+
+  def trusted?
+    @account.trust_level > Account::TRUST_LEVELS[:untrusted]
+  end
+
+  def no_unsolicited_mentions?
+    @status.mentions.all? { |mention| mention.silent? || (!@account.local? && !mention.account.local?) || mention.account.following?(@account) }
+  end
+
+  def solicited_reply?
+    !@status.thread.nil? && @status.thread.mentions.where(account: @account).exists?
+  end
+
+  def nilsimsa_compare_value(first, second)
+    first  = [first].pack('H*')
+    second = [second].pack('H*')
+    bits   = 0
+
+    0.upto(31) do |i|
+      bits += Nilsimsa::POPC[255 & (first[i].ord ^ second[i].ord)].ord
+    end
+
+    128 - bits # -128 <= Nilsimsa Compare Value <= 128
+  end
+
+  def nilsimsa?
+    hashable_text.size > NILSIMSA_MIN_SIZE
+  end
+
+  def other_digests
+    redis.zrange(redis_key, 0, -1)
+  end
+
+  def any_other_digest?(filter_algorithm)
+    other_digests.any? do |record|
+      algorithm, other_digest, status_id = record.split(':')
+
+      next unless algorithm == filter_algorithm
+
+      yield algorithm, other_digest, status_id
+    end
+  end
+
+  def matching_status_ids
+    if nilsimsa?
+      other_digests.select { |record| record.start_with?('nilsimsa') && nilsimsa_compare_value(digest, record.split(':')[1]) >= NILSIMSA_COMPARE_THRESHOLD }.map { |record| record.split(':')[2] }.compact
+    else
+      other_digests.select { |record| record.start_with?('md5') && record.split(':')[1] == digest }.map { |record| record.split(':')[2] }.compact
+    end
+  end
+
+  def redis_key
+    @redis_key ||= "spam_check:#{@account.id}"
+  end
+end
diff --git a/app/models/account.rb b/app/models/account.rb
index d6772eb98..a22b7fd7c 100644
--- a/app/models/account.rb
+++ b/app/models/account.rb
@@ -45,6 +45,7 @@
 #  also_known_as           :string           is an Array
 #  silenced_at             :datetime
 #  suspended_at            :datetime
+#  trust_level             :integer
 #
 
 class Account < ApplicationRecord
@@ -62,6 +63,11 @@ class Account < ApplicationRecord
   include AccountCounters
   include DomainNormalizable
 
+  TRUST_LEVELS = {
+    untrusted: 0,
+    trusted: 1,
+  }.freeze
+
   enum protocol: [:ostatus, :activitypub]
 
   validates :username, presence: true
@@ -163,6 +169,10 @@ class Account < ApplicationRecord
     last_webfingered_at.nil? || last_webfingered_at <= 1.day.ago
   end
 
+  def trust_level
+    self[:trust_level] || 0
+  end
+
   def refresh!
     ResolveAccountService.new.call(acct) unless local?
   end
@@ -171,21 +181,19 @@ class Account < ApplicationRecord
     silenced_at.present?
   end
 
-  def silence!(date = nil)
-    date ||= Time.now.utc
+  def silence!(date = Time.now.utc)
     update!(silenced_at: date)
   end
 
   def unsilence!
-    update!(silenced_at: nil)
+    update!(silenced_at: nil, trust_level: trust_level == TRUST_LEVELS[:untrusted] ? TRUST_LEVELS[:trusted] : trust_level)
   end
 
   def suspended?
     suspended_at.present?
   end
 
-  def suspend!(date = nil)
-    date ||= Time.now.utc
+  def suspend!(date = Time.now.utc)
     transaction do
       user&.disable! if local?
       update!(suspended_at: date)
diff --git a/app/services/remove_status_service.rb b/app/services/remove_status_service.rb
index 6311971ff..a69fce8b8 100644
--- a/app/services/remove_status_service.rb
+++ b/app/services/remove_status_service.rb
@@ -23,6 +23,7 @@ class RemoveStatusService < BaseService
         remove_from_hashtags
         remove_from_public
         remove_from_media if status.media_attachments.any?
+        remove_from_spam_check
 
         @status.destroy!
       else
@@ -142,6 +143,10 @@ class RemoveStatusService < BaseService
     redis.publish('timeline:public:local:media', @payload) if @status.local?
   end
 
+  def remove_from_spam_check
+    redis.zremrangebyscore("spam_check:#{@status.account_id}", @status.id, @status.id)
+  end
+
   def lock_options
     { redis: Redis.current, key: "distribute:#{@status.id}" }
   end
diff --git a/config/locales/en.yml b/config/locales/en.yml
index 00b7d1dbe..89251ad40 100644
--- a/config/locales/en.yml
+++ b/config/locales/en.yml
@@ -875,6 +875,8 @@ en:
     profile: Profile
     relationships: Follows and followers
     two_factor_authentication: Two-factor Auth
+  spam_check:
+    spam_detected_and_silenced: This is an automated report. Spam has been detected and the sender has been silenced automatically. If this is a mistake, please unsilence the account.
   statuses:
     attached:
       description: 'Attached: %{attached}'
diff --git a/db/migrate/20190701022101_add_trust_level_to_accounts.rb b/db/migrate/20190701022101_add_trust_level_to_accounts.rb
new file mode 100644
index 000000000..917486d2e
--- /dev/null
+++ b/db/migrate/20190701022101_add_trust_level_to_accounts.rb
@@ -0,0 +1,5 @@
+class AddTrustLevelToAccounts < ActiveRecord::Migration[5.2]
+  def change
+    add_column :accounts, :trust_level, :integer
+  end
+end
diff --git a/db/schema.rb b/db/schema.rb
index 2e38fb1f2..c7b6b9be6 100644
--- a/db/schema.rb
+++ b/db/schema.rb
@@ -148,6 +148,7 @@ ActiveRecord::Schema.define(version: 2019_07_06_233204) do
     t.string "also_known_as", array: true
     t.datetime "silenced_at"
     t.datetime "suspended_at"
+    t.integer "trust_level"
     t.index "(((setweight(to_tsvector('simple'::regconfig, (display_name)::text), 'A'::\"char\") || setweight(to_tsvector('simple'::regconfig, (username)::text), 'B'::\"char\")) || setweight(to_tsvector('simple'::regconfig, (COALESCE(domain, ''::character varying))::text), 'C'::\"char\")))", name: "search_index", using: :gin
     t.index "lower((username)::text), lower((domain)::text)", name: "index_accounts_on_username_and_domain_lower", unique: true
     t.index ["moved_to_account_id"], name: "index_accounts_on_moved_to_account_id"
diff --git a/spec/lib/spam_check_spec.rb b/spec/lib/spam_check_spec.rb
new file mode 100644
index 000000000..c722dc642
--- /dev/null
+++ b/spec/lib/spam_check_spec.rb
@@ -0,0 +1,160 @@
+require 'rails_helper'
+
+RSpec.describe SpamCheck do
+  let!(:sender) { Fabricate(:account) }
+  let!(:alice) { Fabricate(:account, username: 'alice') }
+  let!(:bob) { Fabricate(:account, username: 'bob') }
+
+  def status_with_html(text, options = {})
+    status = PostStatusService.new.call(sender, { text: text }.merge(options))
+    status.update_columns(text: Formatter.instance.format(status), local: false)
+    status
+  end
+
+  describe '#hashable_text' do
+    it 'removes mentions from HTML for remote statuses' do
+      status = status_with_html('@alice Hello')
+      expect(described_class.new(status).hashable_text).to eq 'hello'
+    end
+
+    it 'removes mentions from text for local statuses' do
+      status = PostStatusService.new.call(alice, text: "Hey @#{sender.username}, how are you?")
+      expect(described_class.new(status).hashable_text).to eq 'hey , how are you?'
+    end
+  end
+
+  describe '#insufficient_data?' do
+    it 'returns true when there is no text' do
+      status = status_with_html('@alice')
+      expect(described_class.new(status).insufficient_data?).to be true
+    end
+
+    it 'returns false when there is text' do
+      status = status_with_html('@alice h')
+      expect(described_class.new(status).insufficient_data?).to be false
+    end
+  end
+
+  describe '#digest' do
+    it 'returns a string' do
+      status = status_with_html('@alice Hello world')
+      expect(described_class.new(status).digest).to be_a String
+    end
+  end
+
+  describe '#spam?' do
+    it 'returns false for a unique status' do
+      status = status_with_html('@alice Hello')
+      expect(described_class.new(status).spam?).to be false
+    end
+
+    it 'returns false for different statuses to the same recipient' do
+      status1 = status_with_html('@alice Hello')
+      described_class.new(status1).remember!
+      status2 = status_with_html('@alice Are you available to talk?')
+      expect(described_class.new(status2).spam?).to be false
+    end
+
+    it 'returns false for statuses with different content warnings' do
+      status1 = status_with_html('@alice Are you available to talk?')
+      described_class.new(status1).remember!
+      status2 = status_with_html('@alice Are you available to talk?', spoiler_text: 'This is a completely different matter than what I was talking about previously, I swear!')
+      expect(described_class.new(status2).spam?).to be false
+    end
+
+    it 'returns false for different statuses to different recipients' do
+      status1 = status_with_html('@alice How is it going?')
+      described_class.new(status1).remember!
+      status2 = status_with_html('@bob Are you okay?')
+      expect(described_class.new(status2).spam?).to be false
+    end
+
+    it 'returns false for very short different statuses to different recipients' do
+      status1 = status_with_html('@alice 🙄')
+      described_class.new(status1).remember!
+      status2 = status_with_html('@bob Huh?')
+      expect(described_class.new(status2).spam?).to be false
+    end
+
+    it 'returns false for statuses with no text' do
+      status1 = status_with_html('@alice')
+      described_class.new(status1).remember!
+      status2 = status_with_html('@bob')
+      expect(described_class.new(status2).spam?).to be false
+    end
+
+    it 'returns true for duplicate statuses to the same recipient' do
+      status1 = status_with_html('@alice Hello')
+      described_class.new(status1).remember!
+      status2 = status_with_html('@alice Hello')
+      expect(described_class.new(status2).spam?).to be true
+    end
+
+    it 'returns true for duplicate statuses to different recipients' do
+      status1 = status_with_html('@alice Hello')
+      described_class.new(status1).remember!
+      status2 = status_with_html('@bob Hello')
+      expect(described_class.new(status2).spam?).to be true
+    end
+
+    it 'returns true for nearly identical statuses with random numbers' do
+      source_text = 'Sodium, atomic number 11, was first isolated by Humphry Davy in 1807. A chemical component of salt, he named it Na in honor of the saltiest region on earth, North America.'
+      status1 = status_with_html('@alice ' + source_text + ' 1234')
+      described_class.new(status1).remember!
+      status2 = status_with_html('@bob ' + source_text + ' 9568')
+      expect(described_class.new(status2).spam?).to be true
+    end
+  end
+
+  describe '#skip?' do
+    it 'returns true when the sender is already silenced' do
+      status = status_with_html('@alice Hello')
+      sender.silence!
+      expect(described_class.new(status).skip?).to be true
+    end
+
+    it 'returns true when the mentioned person follows the sender' do
+      status = status_with_html('@alice Hello')
+      alice.follow!(sender)
+      expect(described_class.new(status).skip?).to be true
+    end
+
+    it 'returns false when even one mentioned person doesn\'t follow the sender' do
+      status = status_with_html('@alice @bob Hello')
+      alice.follow!(sender)
+      expect(described_class.new(status).skip?).to be false
+    end
+
+    it 'returns true when the sender is replying to a status that mentions the sender' do
+      parent = PostStatusService.new.call(alice, text: "Hey @#{sender.username}, how are you?")
+      status = status_with_html('@alice @bob Hello', thread: parent)
+      expect(described_class.new(status).skip?).to be true
+    end
+  end
+
+  describe '#remember!' do
+    pending
+  end
+
+  describe '#flag!' do
+    let!(:status1) { status_with_html('@alice General Kenobi you are a bold one') }
+    let!(:status2) { status_with_html('@alice @bob General Kenobi, you are a bold one') }
+
+    before do
+      described_class.new(status1).remember!
+      described_class.new(status2).flag!
+    end
+
+    it 'silences the account' do
+      expect(sender.silenced?).to be true
+    end
+
+    it 'creates a report about the account' do
+      expect(sender.targeted_reports.unresolved.count).to eq 1
+    end
+
+    it 'attaches both matching statuses to the report' do
+      expect(sender.targeted_reports.first.status_ids).to include(status1.id, status2.id)
+    end
+  end
+end
-- 
cgit 


From 5bfe1e1f0517a23637a1a132dbf0b62fd29982bc Mon Sep 17 00:00:00 2001
From: Eugen Rochko <eugen@zeonfederated.com>
Date: Thu, 18 Jul 2019 03:02:15 +0200
Subject: Change language detection to include hashtags as words (#11341)

---
 app/lib/language_detector.rb       | 2 +-
 spec/lib/language_detector_spec.rb | 6 +++---
 2 files changed, 4 insertions(+), 4 deletions(-)

(limited to 'spec')

diff --git a/app/lib/language_detector.rb b/app/lib/language_detector.rb
index 1e90af42d..6f9511a54 100644
--- a/app/lib/language_detector.rb
+++ b/app/lib/language_detector.rb
@@ -69,7 +69,7 @@ class LanguageDetector
     new_text = remove_html(text)
     new_text.gsub!(FetchLinkCardService::URL_PATTERN, '')
     new_text.gsub!(Account::MENTION_RE, '')
-    new_text.gsub!(Tag::HASHTAG_RE, '')
+    new_text.gsub!(Tag::HASHTAG_RE) { |string| string.gsub(/[#_]/, '#' => '', '_' => ' ').gsub(/[a-z][A-Z]|[a-zA-Z][\d]/) { |s| s.insert(1, ' ') }.downcase }
     new_text.gsub!(/:#{CustomEmoji::SHORTCODE_RE_FRAGMENT}:/, '')
     new_text.gsub!(/\s+/, ' ')
     new_text
diff --git a/spec/lib/language_detector_spec.rb b/spec/lib/language_detector_spec.rb
index 0cb70605a..b7ba0f6c4 100644
--- a/spec/lib/language_detector_spec.rb
+++ b/spec/lib/language_detector_spec.rb
@@ -32,11 +32,11 @@ describe LanguageDetector do
       expect(result).to eq 'Our website is and also'
     end
 
-    it 'strips #hashtags from strings before detection' do
-      string = 'Hey look at all the #animals and #fish'
+    it 'converts #hashtags back to normal text before detection' do
+      string = 'Hey look at all the #animals and #FishAndChips'
 
       result = described_class.instance.send(:prepare_text, string)
-      expect(result).to eq 'Hey look at all the and'
+      expect(result).to eq 'Hey look at all the animals and fish and chips'
     end
   end
 
-- 
cgit 


From 84e988479e956bcf522072879da936b100d1d46b Mon Sep 17 00:00:00 2001
From: Eugen Rochko <eugen@zeonfederated.com>
Date: Thu, 18 Jul 2019 03:02:56 +0200
Subject: Fix only one middle dot being recognized in hashtags (#11345)

Fix #10934
---
 app/models/tag.rb       |  2 +-
 spec/models/tag_spec.rb | 38 +++++++++++++++++++++++++++++++++++++-
 2 files changed, 38 insertions(+), 2 deletions(-)

(limited to 'spec')

diff --git a/app/models/tag.rb b/app/models/tag.rb
index 7db76d157..01bace2bb 100644
--- a/app/models/tag.rb
+++ b/app/models/tag.rb
@@ -17,7 +17,7 @@ class Tag < ApplicationRecord
   has_many :featured_tags, dependent: :destroy, inverse_of: :tag
   has_one :account_tag_stat, dependent: :destroy
 
-  HASHTAG_NAME_RE = '[[:word:]_]*[[:alpha:]_·][[:word:]_]*'
+  HASHTAG_NAME_RE = '[[:word:]_][[:word:]_]*[[:alpha:]_·]*[[:word:]_·]*[[:word:]_]'
   HASHTAG_RE = /(?:^|[^\/\)\w])#(#{HASHTAG_NAME_RE})/i
 
   validates :name, presence: true, uniqueness: true, format: { with: /\A#{HASHTAG_NAME_RE}\z/i }
diff --git a/spec/models/tag_spec.rb b/spec/models/tag_spec.rb
index 1ca50cc29..161862392 100644
--- a/spec/models/tag_spec.rb
+++ b/spec/models/tag_spec.rb
@@ -31,7 +31,43 @@ RSpec.describe Tag, type: :model do
     end
 
     it 'matches #ａｅｓｔｈｅｔｉｃ' do
-      expect(subject.match('this is #ａｅｓｔｈｅｔｉｃ')).to_not be_nil
+      expect(subject.match('this is #ａｅｓｔｈｅｔｉｃ').to_s).to eq ' #ａｅｓｔｈｅｔｉｃ'
+    end
+
+    it 'matches digits at the start' do
+      expect(subject.match('hello #3d').to_s).to eq ' #3d'
+    end
+
+    it 'matches digits in the middle' do
+      expect(subject.match('hello #l33ts35k').to_s).to eq ' #l33ts35k'
+    end
+
+    it 'matches digits at the end' do
+      expect(subject.match('hello #world2016').to_s).to eq ' #world2016'
+    end
+
+    it 'matches underscores at the beginning' do
+      expect(subject.match('hello #_test').to_s).to eq ' #_test'
+    end
+
+    it 'matches underscores at the end' do
+      expect(subject.match('hello #test_').to_s).to eq ' #test_'
+    end
+
+    it 'matches underscores in the middle' do
+      expect(subject.match('hello #one_two_three').to_s).to eq ' #one_two_three'
+    end
+
+    it 'matches middle dots' do
+      expect(subject.match('hello #one·two·three').to_s).to eq ' #one·two·three'
+    end
+
+    it 'does not match middle dots at the start' do
+      expect(subject.match('hello #·one·two·three')).to be_nil
+    end
+
+    it 'does not match middle dots at the end' do
+      expect(subject.match('hello #one·two·three·').to_s).to eq ' #one·two·three'
     end
   end
 
-- 
cgit 


From 730c4053d642024b9949d72c8a9f1873532c6212 Mon Sep 17 00:00:00 2001
From: ThibG <thib@sitedethib.com>
Date: Fri, 19 Jul 2019 01:44:42 +0200
Subject: Add ActivityPub actor representing the entire server (#11321)

* Add support for an instance actor

* Skip username validation for local Application accounts

* Add migration script to create instance actor

* Make Codeclimate happy

* Switch to id -99 for instance actor

* Remove unused `icon` and `image` attributes from instance actor

* Use if/elsif/else instead of return + ternary operator

* Add instance actor to fresh installs

* Use instance actor as instance representative

Use instance actor for forwarding reports, relay operations, and spam
auto-reporting.

* Seed database in test environment

* Fix single-user mode

* Fix tests

* Fix specs to accomodate for an extra `Account`

* Auto-reject follows on instance actor

Following an instance actor might make sense, but we are not handling that
right now, so auto-reject.

* Fix webfinger lookup and serialization for instance actor

* Rename instance actor

* Make it clear in the HTML view that the instance actor should not be blocked

* Raise cache time for instance actor as there's no dynamic content

* Re-use /about/more with a flash message for instance actor profile
---
 app/controllers/about_controller.rb                |  4 +-
 app/controllers/application_controller.rb          |  2 +-
 app/controllers/home_controller.rb                 |  2 +-
 app/controllers/instance_actors_controller.rb      | 20 ++++++++
 app/javascript/styles/mastodon/containers.scss     |  4 ++
 app/lib/activitypub/activity/follow.rb             |  2 +-
 app/lib/activitypub/tag_manager.rb                 |  5 +-
 app/lib/webfinger_resource.rb                      |  6 +++
 app/models/account.rb                              |  8 ++-
 app/models/concerns/account_finder_concern.rb      |  2 +-
 app/serializers/activitypub/actor_serializer.rb    | 14 ++++--
 app/serializers/webfinger_serializer.rb            | 25 +++++++---
 app/views/about/more.html.haml                     |  2 +
 app/views/well_known/webfinger/show.xml.ruby       | 57 ++++++++++++++--------
 config/locales/en.yml                              |  3 ++
 config/routes.rb                                   |  4 ++
 db/migrate/20190715164535_add_instance_actor.rb    |  9 ++++
 db/schema.rb                                       |  2 +-
 db/seeds.rb                                        |  4 +-
 spec/models/account_spec.rb                        | 12 ++---
 spec/services/fetch_remote_account_service_spec.rb |  1 -
 spec/services/fetch_resource_service_spec.rb       |  4 +-
 spec/spec_helper.rb                                |  1 +
 23 files changed, 141 insertions(+), 52 deletions(-)
 create mode 100644 app/controllers/instance_actors_controller.rb
 create mode 100644 db/migrate/20190715164535_add_instance_actor.rb

(limited to 'spec')

diff --git a/app/controllers/about_controller.rb b/app/controllers/about_controller.rb
index 52fb1dc1b..33bac9bbc 100644
--- a/app/controllers/about_controller.rb
+++ b/app/controllers/about_controller.rb
@@ -11,7 +11,9 @@ class AboutController < ApplicationController
 
   def show; end
 
-  def more; end
+  def more
+    flash.now[:notice] = I18n.t('about.instance_actor_flash') if params[:instance_actor]
+  end
 
   def terms; end
 
diff --git a/app/controllers/application_controller.rb b/app/controllers/application_controller.rb
index 26f3b1def..51e9764d4 100644
--- a/app/controllers/application_controller.rb
+++ b/app/controllers/application_controller.rb
@@ -91,7 +91,7 @@ class ApplicationController < ActionController::Base
   end
 
   def single_user_mode?
-    @single_user_mode ||= Rails.configuration.x.single_user_mode && Account.exists?
+    @single_user_mode ||= Rails.configuration.x.single_user_mode && Account.where('id > 0').exists?
   end
 
   def use_seamless_external_login?
diff --git a/app/controllers/home_controller.rb b/app/controllers/home_controller.rb
index d1c525134..42493cd78 100644
--- a/app/controllers/home_controller.rb
+++ b/app/controllers/home_controller.rb
@@ -58,7 +58,7 @@ class HomeController < ApplicationController
     if request.path.start_with?('/web')
       new_user_session_path
     elsif single_user_mode?
-      short_account_path(Account.local.without_suspended.first)
+      short_account_path(Account.local.without_suspended.where('id > 0').first)
     else
       about_path
     end
diff --git a/app/controllers/instance_actors_controller.rb b/app/controllers/instance_actors_controller.rb
new file mode 100644
index 000000000..41f33602e
--- /dev/null
+++ b/app/controllers/instance_actors_controller.rb
@@ -0,0 +1,20 @@
+# frozen_string_literal: true
+
+class InstanceActorsController < ApplicationController
+  include AccountControllerConcern
+
+  def show
+    expires_in 10.minutes, public: true
+    render json: @account, content_type: 'application/activity+json', serializer: ActivityPub::ActorSerializer, adapter: ActivityPub::Adapter, fields: restrict_fields_to
+  end
+
+  private
+
+  def set_account
+    @account = Account.find(-99)
+  end
+
+  def restrict_fields_to
+    %i(id type preferred_username inbox public_key endpoints url manually_approves_followers)
+  end
+end
diff --git a/app/javascript/styles/mastodon/containers.scss b/app/javascript/styles/mastodon/containers.scss
index 3564bf07b..2b6794ee2 100644
--- a/app/javascript/styles/mastodon/containers.scss
+++ b/app/javascript/styles/mastodon/containers.scss
@@ -145,6 +145,10 @@
     min-height: 100%;
   }
 
+  .flash-message {
+    margin-bottom: 10px;
+  }
+
   @media screen and (max-width: 738px) {
     grid-template-columns: minmax(0, 50%) minmax(0, 50%);
 
diff --git a/app/lib/activitypub/activity/follow.rb b/app/lib/activitypub/activity/follow.rb
index 3eb88339a..28f1da19f 100644
--- a/app/lib/activitypub/activity/follow.rb
+++ b/app/lib/activitypub/activity/follow.rb
@@ -8,7 +8,7 @@ class ActivityPub::Activity::Follow < ActivityPub::Activity
 
     return if target_account.nil? || !target_account.local? || delete_arrived_first?(@json['id']) || @account.requested?(target_account)
 
-    if target_account.blocking?(@account) || target_account.domain_blocking?(@account.domain) || target_account.moved?
+    if target_account.blocking?(@account) || target_account.domain_blocking?(@account.domain) || target_account.moved? || target_account.instance_actor?
       reject_follow_request!(target_account)
       return
     end
diff --git a/app/lib/activitypub/tag_manager.rb b/app/lib/activitypub/tag_manager.rb
index 4d452f290..512272dbe 100644
--- a/app/lib/activitypub/tag_manager.rb
+++ b/app/lib/activitypub/tag_manager.rb
@@ -17,7 +17,7 @@ class ActivityPub::TagManager
 
     case target.object_type
     when :person
-      short_account_url(target)
+      target.instance_actor? ? about_more_url(instance_actor: true) : short_account_url(target)
     when :note, :comment, :activity
       return activity_account_status_url(target.account, target) if target.reblog?
       short_account_status_url(target.account, target)
@@ -29,7 +29,7 @@ class ActivityPub::TagManager
 
     case target.object_type
     when :person
-      account_url(target)
+      target.instance_actor? ? instance_actor_url : account_url(target)
     when :note, :comment, :activity
       return activity_account_status_url(target.account, target) if target.reblog?
       account_status_url(target.account, target)
@@ -119,6 +119,7 @@ class ActivityPub::TagManager
 
   def uri_to_local_id(uri, param = :id)
     path_params = Rails.application.routes.recognize_path(uri)
+    path_params[:username] = Rails.configuration.x.local_domain if path_params[:controller] == 'instance_actors'
     path_params[param]
   end
 
diff --git a/app/lib/webfinger_resource.rb b/app/lib/webfinger_resource.rb
index a54a702a2..22d78874a 100644
--- a/app/lib/webfinger_resource.rb
+++ b/app/lib/webfinger_resource.rb
@@ -23,11 +23,17 @@ class WebfingerResource
   def username_from_url
     if account_show_page?
       path_params[:username]
+    elsif instance_actor_page?
+      Rails.configuration.x.local_domain
     else
       raise ActiveRecord::RecordNotFound
     end
   end
 
+  def instance_actor_page?
+    path_params[:controller] == 'instance_actors'
+  end
+
   def account_show_page?
     path_params[:controller] == 'accounts' && path_params[:action] == 'show'
   end
diff --git a/app/models/account.rb b/app/models/account.rb
index adf4586fa..ccd116d6e 100644
--- a/app/models/account.rb
+++ b/app/models/account.rb
@@ -77,7 +77,7 @@ class Account < ApplicationRecord
   validates :username, format: { with: /\A#{USERNAME_RE}\z/i }, if: -> { !local? && will_save_change_to_username? }
 
   # Local user validations
-  validates :username, format: { with: /\A[a-z0-9_]+\z/i }, length: { maximum: 30 }, if: -> { local? && will_save_change_to_username? }
+  validates :username, format: { with: /\A[a-z0-9_]+\z/i }, length: { maximum: 30 }, if: -> { local? && will_save_change_to_username? && actor_type != 'Application' }
   validates_with UniqueUsernameValidator, if: -> { local? && will_save_change_to_username? }
   validates_with UnreservedUsernameValidator, if: -> { local? && will_save_change_to_username? }
   validates :display_name, length: { maximum: 30 }, if: -> { local? && will_save_change_to_display_name? }
@@ -139,6 +139,10 @@ class Account < ApplicationRecord
     %w(Application Service).include? actor_type
   end
 
+  def instance_actor?
+    id == -99
+  end
+
   alias bot bot?
 
   def bot=(val)
@@ -498,7 +502,7 @@ class Account < ApplicationRecord
   end
 
   def generate_keys
-    return unless local? && !Rails.env.test?
+    return unless local? && private_key.blank? && public_key.blank?
 
     keypair = OpenSSL::PKey::RSA.new(2048)
     self.private_key = keypair.to_pem
diff --git a/app/models/concerns/account_finder_concern.rb b/app/models/concerns/account_finder_concern.rb
index ccd7bfa12..a54c2174d 100644
--- a/app/models/concerns/account_finder_concern.rb
+++ b/app/models/concerns/account_finder_concern.rb
@@ -13,7 +13,7 @@ module AccountFinderConcern
     end
 
     def representative
-      find_local(Setting.site_contact_username.strip.gsub(/\A@/, '')) || Account.local.without_suspended.first
+      Account.find(-99)
     end
 
     def find_local(username)
diff --git a/app/serializers/activitypub/actor_serializer.rb b/app/serializers/activitypub/actor_serializer.rb
index 0644219fb..0bd7aed2e 100644
--- a/app/serializers/activitypub/actor_serializer.rb
+++ b/app/serializers/activitypub/actor_serializer.rb
@@ -39,11 +39,17 @@ class ActivityPub::ActorSerializer < ActivityPub::Serializer
   delegate :moved?, to: :object
 
   def id
-    account_url(object)
+    object.instance_actor? ? instance_actor_url : account_url(object)
   end
 
   def type
-    object.bot? ? 'Service' : 'Person'
+    if object.instance_actor?
+      'Application'
+    elsif object.bot?
+      'Service'
+    else
+      'Person'
+    end
   end
 
   def following
@@ -55,7 +61,7 @@ class ActivityPub::ActorSerializer < ActivityPub::Serializer
   end
 
   def inbox
-    account_inbox_url(object)
+    object.instance_actor? ? instance_actor_inbox_url : account_inbox_url(object)
   end
 
   def outbox
@@ -95,7 +101,7 @@ class ActivityPub::ActorSerializer < ActivityPub::Serializer
   end
 
   def url
-    short_account_url(object)
+    object.instance_actor? ? about_more_url(instance_actor: true) : short_account_url(object)
   end
 
   def avatar_exists?
diff --git a/app/serializers/webfinger_serializer.rb b/app/serializers/webfinger_serializer.rb
index f4af21551..008d0c182 100644
--- a/app/serializers/webfinger_serializer.rb
+++ b/app/serializers/webfinger_serializer.rb
@@ -10,15 +10,26 @@ class WebfingerSerializer < ActiveModel::Serializer
   end
 
   def aliases
-    [short_account_url(object), account_url(object)]
+    if object.instance_actor?
+      [instance_actor_url]
+    else
+      [short_account_url(object), account_url(object)]
+    end
   end
 
   def links
-    [
-      { rel: 'http://webfinger.net/rel/profile-page', type: 'text/html', href: short_account_url(object) },
-      { rel: 'http://schemas.google.com/g/2010#updates-from', type: 'application/atom+xml', href: account_url(object, format: 'atom') },
-      { rel: 'self', type: 'application/activity+json', href: account_url(object) },
-      { rel: 'http://ostatus.org/schema/1.0/subscribe', template: "#{authorize_interaction_url}?uri={uri}" },
-    ]
+    if object.instance_actor?
+      [
+        { rel: 'http://webfinger.net/rel/profile-page', type: 'text/html', href: about_more_url(instance_actor: true) },
+        { rel: 'self', type: 'application/activity+json', href: instance_actor_url },
+      ]
+    else
+      [
+        { rel: 'http://webfinger.net/rel/profile-page', type: 'text/html', href: short_account_url(object) },
+        { rel: 'http://schemas.google.com/g/2010#updates-from', type: 'application/atom+xml', href: account_url(object, format: 'atom') },
+        { rel: 'self', type: 'application/activity+json', href: account_url(object) },
+        { rel: 'http://ostatus.org/schema/1.0/subscribe', template: "#{authorize_interaction_url}?uri={uri}" },
+      ]
+    end
   end
 end
diff --git a/app/views/about/more.html.haml b/app/views/about/more.html.haml
index b248ed1d2..21431ef8e 100644
--- a/app/views/about/more.html.haml
+++ b/app/views/about/more.html.haml
@@ -43,5 +43,7 @@
           = mail_to @instance_presenter.site_contact_email, nil, title: @instance_presenter.site_contact_email
 
   .column-3
+    = render 'application/flashes'
+
     .box-widget
       .rich-formatting= @instance_presenter.site_extended_description.html_safe.presence || t('about.extended_description_html')
diff --git a/app/views/well_known/webfinger/show.xml.ruby b/app/views/well_known/webfinger/show.xml.ruby
index ae80df9d2..f5a54052a 100644
--- a/app/views/well_known/webfinger/show.xml.ruby
+++ b/app/views/well_known/webfinger/show.xml.ruby
@@ -4,30 +4,47 @@ doc << Ox::Element.new('XRD').tap do |xrd|
   xrd['xmlns'] = 'http://docs.oasis-open.org/ns/xri/xrd-1.0'
 
   xrd << (Ox::Element.new('Subject') << @account.to_webfinger_s)
-  xrd << (Ox::Element.new('Alias') << short_account_url(@account))
-  xrd << (Ox::Element.new('Alias') << account_url(@account))
 
-  xrd << Ox::Element.new('Link').tap do |link|
-    link['rel']      = 'http://webfinger.net/rel/profile-page'
-    link['type']     = 'text/html'
-    link['href']     = short_account_url(@account)
-  end
+  if @account.instance_actor?
+    xrd << (Ox::Element.new('Alias') << instance_actor_url)
 
-  xrd << Ox::Element.new('Link').tap do |link|
-    link['rel']      = 'http://schemas.google.com/g/2010#updates-from'
-    link['type']     = 'application/atom+xml'
-    link['href']     = account_url(@account, format: 'atom')
-  end
+    xrd << Ox::Element.new('Link').tap do |link|
+      link['rel']      = 'http://webfinger.net/rel/profile-page'
+      link['type']     = 'text/html'
+      link['href']     = about_more_url(instance_actor: true)
+    end
 
-  xrd << Ox::Element.new('Link').tap do |link|
-    link['rel']      = 'self'
-    link['type']     = 'application/activity+json'
-    link['href']     = account_url(@account)
-  end
+    xrd << Ox::Element.new('Link').tap do |link|
+      link['rel']      = 'self'
+      link['type']     = 'application/activity+json'
+      link['href']     = instance_actor_url
+    end
+  else
+    xrd << (Ox::Element.new('Alias') << short_account_url(@account))
+    xrd << (Ox::Element.new('Alias') << account_url(@account))
+
+    xrd << Ox::Element.new('Link').tap do |link|
+      link['rel']      = 'http://webfinger.net/rel/profile-page'
+      link['type']     = 'text/html'
+      link['href']     = short_account_url(@account)
+    end
+
+    xrd << Ox::Element.new('Link').tap do |link|
+      link['rel']      = 'http://schemas.google.com/g/2010#updates-from'
+      link['type']     = 'application/atom+xml'
+      link['href']     = account_url(@account, format: 'atom')
+    end
+
+    xrd << Ox::Element.new('Link').tap do |link|
+      link['rel']      = 'self'
+      link['type']     = 'application/activity+json'
+      link['href']     = account_url(@account)
+    end
 
-  xrd << Ox::Element.new('Link').tap do |link|
-    link['rel']      = 'http://ostatus.org/schema/1.0/subscribe'
-    link['template'] = "#{authorize_interaction_url}?acct={uri}"
+    xrd << Ox::Element.new('Link').tap do |link|
+      link['rel']      = 'http://ostatus.org/schema/1.0/subscribe'
+      link['template'] = "#{authorize_interaction_url}?acct={uri}"
+    end
   end
 end
 
diff --git a/config/locales/en.yml b/config/locales/en.yml
index 4e252945f..89c52b84a 100644
--- a/config/locales/en.yml
+++ b/config/locales/en.yml
@@ -24,6 +24,9 @@ en:
     generic_description: "%{domain} is one server in the network"
     get_apps: Try a mobile app
     hosted_on: Mastodon hosted on %{domain}
+    instance_actor_flash: |
+      This account is a virtual actor used to represent the server itself and not any individual user.
+      It is used for federation purposes and should not be blocked unless you want to block the whole instance, in which case you should use a domain block.
     learn_more: Learn more
     privacy_policy: Privacy policy
     see_whats_happening: See what's happening
diff --git a/config/routes.rb b/config/routes.rb
index 95f8a39ad..27b536641 100644
--- a/config/routes.rb
+++ b/config/routes.rb
@@ -28,6 +28,10 @@ Rails.application.routes.draw do
   get 'intent', to: 'intents#show'
   get 'custom.css', to: 'custom_css#show', as: :custom_css
 
+  resource :instance_actor, path: 'actor', only: [:show] do
+    resource :inbox, only: [:create], module: :activitypub
+  end
+
   devise_scope :user do
     get '/invite/:invite_code', to: 'auth/registrations#new', as: :public_invite
     match '/auth/finish_signup' => 'auth/confirmations#finish_signup', via: [:get, :patch], as: :finish_signup
diff --git a/db/migrate/20190715164535_add_instance_actor.rb b/db/migrate/20190715164535_add_instance_actor.rb
new file mode 100644
index 000000000..a26d54949
--- /dev/null
+++ b/db/migrate/20190715164535_add_instance_actor.rb
@@ -0,0 +1,9 @@
+class AddInstanceActor < ActiveRecord::Migration[5.2]
+  def up
+    Account.create!(id: -99, actor_type: 'Application', locked: true, username: Rails.configuration.x.local_domain)
+  end
+
+  def down
+    Account.find_by(id: -99, actor_type: 'Application').destroy!
+  end
+end
diff --git a/db/schema.rb b/db/schema.rb
index c7b6b9be6..a6a14827b 100644
--- a/db/schema.rb
+++ b/db/schema.rb
@@ -10,7 +10,7 @@
 #
 # It's strongly recommended that you check this file into your version control system.
 
-ActiveRecord::Schema.define(version: 2019_07_06_233204) do
+ActiveRecord::Schema.define(version: 2019_07_15_164535) do
 
   # These are extensions that must be enabled in order to support this database
   enable_extension "plpgsql"
diff --git a/db/seeds.rb b/db/seeds.rb
index 9a6e9dd78..5f43fbac8 100644
--- a/db/seeds.rb
+++ b/db/seeds.rb
@@ -1,7 +1,9 @@
 Doorkeeper::Application.create!(name: 'Web', superapp: true, redirect_uri: Doorkeeper.configuration.native_redirect_uri, scopes: 'read write follow')
 
+domain = ENV['LOCAL_DOMAIN'] || Rails.configuration.x.local_domain
+Account.create!(id: -99, actor_type: 'Application', locked: true, username: domain)
+
 if Rails.env.development?
-  domain = ENV['LOCAL_DOMAIN'] || Rails.configuration.x.local_domain
   admin  = Account.where(username: 'admin').first_or_initialize(username: 'admin')
   admin.save(validate: false)
   User.where(email: "admin@#{domain}").first_or_initialize(email: "admin@#{domain}", password: 'mastodonadmin', password_confirmation: 'mastodonadmin', confirmed_at: Time.now.utc, admin: true, account: admin, agreement: true, approved: true).save!
diff --git a/spec/models/account_spec.rb b/spec/models/account_spec.rb
index ce9ea250d..6495a6193 100644
--- a/spec/models/account_spec.rb
+++ b/spec/models/account_spec.rb
@@ -450,7 +450,7 @@ RSpec.describe Account, type: :model do
   describe '.domains' do
     it 'returns domains' do
       Fabricate(:account, domain: 'domain')
-      expect(Account.domains).to match_array(['domain'])
+      expect(Account.remote.domains).to match_array(['domain'])
     end
   end
 
@@ -665,7 +665,7 @@ RSpec.describe Account, type: :model do
           { username: 'b', domain: 'b' },
         ].map(&method(:Fabricate).curry(2).call(:account))
 
-        expect(Account.alphabetic).to eq matches
+        expect(Account.where('id > 0').alphabetic).to eq matches
       end
     end
 
@@ -732,7 +732,7 @@ RSpec.describe Account, type: :model do
         2.times { Fabricate(:account, domain: 'example.com') }
         Fabricate(:account, domain: 'example2.com')
 
-        results = Account.by_domain_accounts
+        results = Account.where('id > 0').by_domain_accounts
         expect(results.length).to eq 2
         expect(results.first.domain).to eq 'example.com'
         expect(results.first.accounts_count).to eq 2
@@ -745,7 +745,7 @@ RSpec.describe Account, type: :model do
       it 'returns an array of accounts who do not have a domain' do
         account_1 = Fabricate(:account, domain: nil)
         account_2 = Fabricate(:account, domain: 'example.com')
-        expect(Account.local).to match_array([account_1])
+        expect(Account.where('id > 0').local).to match_array([account_1])
       end
     end
 
@@ -756,14 +756,14 @@ RSpec.describe Account, type: :model do
           matches[index] = Fabricate(:account, domain: matches[index])
         end
 
-        expect(Account.partitioned).to match_array(matches)
+        expect(Account.where('id > 0').partitioned).to match_array(matches)
       end
     end
 
     describe 'recent' do
       it 'returns a relation of accounts sorted by recent creation' do
         matches = 2.times.map { Fabricate(:account) }
-        expect(Account.recent).to match_array(matches)
+        expect(Account.where('id > 0').recent).to match_array(matches)
       end
     end
 
diff --git a/spec/services/fetch_remote_account_service_spec.rb b/spec/services/fetch_remote_account_service_spec.rb
index b37445861..ee7325be2 100644
--- a/spec/services/fetch_remote_account_service_spec.rb
+++ b/spec/services/fetch_remote_account_service_spec.rb
@@ -4,7 +4,6 @@ RSpec.describe FetchRemoteAccountService, type: :service do
   let(:url) { 'https://example.com/alice' }
   let(:prefetched_body) { nil }
   let(:protocol) { :ostatus }
-  let!(:representative) { Fabricate(:account) }
 
   subject { FetchRemoteAccountService.new.call(url, prefetched_body, protocol) }
 
diff --git a/spec/services/fetch_resource_service_spec.rb b/spec/services/fetch_resource_service_spec.rb
index 98630966b..f836147d3 100644
--- a/spec/services/fetch_resource_service_spec.rb
+++ b/spec/services/fetch_resource_service_spec.rb
@@ -1,8 +1,6 @@
 require 'rails_helper'
 
 RSpec.describe FetchResourceService, type: :service do
-  let!(:representative) { Fabricate(:account) }
-
   describe '#call' do
     let(:url) { 'http://example.com' }
 
@@ -60,7 +58,7 @@ RSpec.describe FetchResourceService, type: :service do
 
       it 'signs request' do
         subject
-        expect(a_request(:get, url).with(headers: { 'Signature' => /keyId="#{Regexp.escape(ActivityPub::TagManager.instance.uri_for(representative) + '#main-key')}"/ })).to have_been_made
+        expect(a_request(:get, url).with(headers: { 'Signature' => /keyId="#{Regexp.escape(ActivityPub::TagManager.instance.uri_for(Account.representative) + '#main-key')}"/ })).to have_been_made
       end
 
       context 'when content type is application/atom+xml' do
diff --git a/spec/spec_helper.rb b/spec/spec_helper.rb
index 0cd1f91d0..45ba1bbd9 100644
--- a/spec/spec_helper.rb
+++ b/spec/spec_helper.rb
@@ -27,6 +27,7 @@ RSpec.configure do |config|
   end
 
   config.before :suite do
+    Rails.application.load_seed
     Chewy.strategy(:bypass)
   end
 
-- 
cgit 


From fda437a02088ac114fecb69e3b1e52f495a2dd9a Mon Sep 17 00:00:00 2001
From: ThibG <thib@sitedethib.com>
Date: Fri, 19 Jul 2019 01:44:58 +0200
Subject: Fix sanitizing lists contents (#11354)

* Add test

* Fix code for sanitizing nested lists stripping all tags
---
 app/lib/sanitize_config.rb       | 2 ++
 spec/lib/sanitize_config_spec.rb | 4 ++++
 2 files changed, 6 insertions(+)

(limited to 'spec')

diff --git a/app/lib/sanitize_config.rb b/app/lib/sanitize_config.rb
index e82a2a33a..aba8ce9f6 100644
--- a/app/lib/sanitize_config.rb
+++ b/app/lib/sanitize_config.rb
@@ -25,6 +25,8 @@ class Sanitize
       case env[:node_name]
       when 'li'
         env[:node].traverse do |node|
+          next unless %w(p ul ol li).include?(node.name)
+
           node.add_next_sibling('<br>') if node.next_sibling
           node.replace(node.children) unless node.text?
         end
diff --git a/spec/lib/sanitize_config_spec.rb b/spec/lib/sanitize_config_spec.rb
index bb3cf6f0b..54bd8693c 100644
--- a/spec/lib/sanitize_config_spec.rb
+++ b/spec/lib/sanitize_config_spec.rb
@@ -22,5 +22,9 @@ describe Sanitize::Config do
     it 'converts ul inside ul' do
       expect(Sanitize.fragment('<ul><li>Foo</li><li><ul><li>Bar</li><li>Baz</li></ul></li></ul>', subject)).to eq '<p>Foo<br>Bar<br>Baz</p>'
     end
+
+    it 'keep links in lists' do
+      expect(Sanitize.fragment('<p>Check out:</p><ul><li><a href="https://joinmastodon.org" rel="nofollow noopener" target="_blank">joinmastodon.org</a></li><li>Bar</li></ul>', subject)).to eq '<p>Check out:</p><p><a href="https://joinmastodon.org" rel="nofollow noopener" target="_blank">joinmastodon.org</a><br>Bar</p>'
+    end
   end
 end
-- 
cgit 


From 3407ae8683b36fd3514aa518b5b1634d0e88d0c7 Mon Sep 17 00:00:00 2001
From: Thibaut Girka <thib@sitedethib.com>
Date: Fri, 19 Jul 2019 19:02:05 +0200
Subject: Fix sanitizer text case for glitch-soc, which preserves lists

---
 spec/lib/sanitize_config_spec.rb | 4 ----
 1 file changed, 4 deletions(-)

(limited to 'spec')

diff --git a/spec/lib/sanitize_config_spec.rb b/spec/lib/sanitize_config_spec.rb
index faefac803..c5143bcef 100644
--- a/spec/lib/sanitize_config_spec.rb
+++ b/spec/lib/sanitize_config_spec.rb
@@ -14,9 +14,5 @@ describe Sanitize::Config do
     it 'keeps ul' do
       expect(Sanitize.fragment('<p>Check out:</p><ul><li>Foo</li><li>Bar</li></ul>', subject)).to eq '<p>Check out:</p><ul><li>Foo</li><li>Bar</li></ul>'
     end
-
-    it 'keep links in lists' do
-      expect(Sanitize.fragment('<p>Check out:</p><ul><li><a href="https://joinmastodon.org" rel="nofollow noopener" target="_blank">joinmastodon.org</a></li><li>Bar</li></ul>', subject)).to eq '<p>Check out:</p><p><a href="https://joinmastodon.org" rel="nofollow noopener" target="_blank">joinmastodon.org</a><br>Bar</p>'
-    end
   end
 end
-- 
cgit 


From c37c1da41e1c490771a409ad1b12f3db55af4cee Mon Sep 17 00:00:00 2001
From: ThibG <thib@sitedethib.com>
Date: Fri, 19 Jul 2019 23:22:35 +0200
Subject: Disallow numeric-only hashtags (#11363)

* Add spec covering numeric-only hashtags

* Fix hashtag regex
---
 app/models/tag.rb       | 4 ++--
 spec/models/tag_spec.rb | 4 ++++
 2 files changed, 6 insertions(+), 2 deletions(-)

(limited to 'spec')

diff --git a/app/models/tag.rb b/app/models/tag.rb
index 01bace2bb..b371d59c1 100644
--- a/app/models/tag.rb
+++ b/app/models/tag.rb
@@ -17,10 +17,10 @@ class Tag < ApplicationRecord
   has_many :featured_tags, dependent: :destroy, inverse_of: :tag
   has_one :account_tag_stat, dependent: :destroy
 
-  HASHTAG_NAME_RE = '[[:word:]_][[:word:]_]*[[:alpha:]_·]*[[:word:]_·]*[[:word:]_]'
+  HASHTAG_NAME_RE = '([[:word:]_][[:word:]_·]*[[:alpha:]_·][[:word:]_·]*[[:word:]_])|([[:word:]_]*[[:alpha:]][[:word:]_]*)'
   HASHTAG_RE = /(?:^|[^\/\)\w])#(#{HASHTAG_NAME_RE})/i
 
-  validates :name, presence: true, uniqueness: true, format: { with: /\A#{HASHTAG_NAME_RE}\z/i }
+  validates :name, presence: true, uniqueness: true, format: { with: /\A(#{HASHTAG_NAME_RE})\z/i }
 
   scope :discoverable, -> { joins(:account_tag_stat).where(AccountTagStat.arel_table[:accounts_count].gt(0)).where(account_tag_stats: { hidden: false }).order(Arel.sql('account_tag_stats.accounts_count desc')) }
   scope :hidden, -> { where(account_tag_stats: { hidden: true }) }
diff --git a/spec/models/tag_spec.rb b/spec/models/tag_spec.rb
index 161862392..9a30ceaa5 100644
--- a/spec/models/tag_spec.rb
+++ b/spec/models/tag_spec.rb
@@ -69,6 +69,10 @@ RSpec.describe Tag, type: :model do
     it 'does not match middle dots at the end' do
       expect(subject.match('hello #one·two·three·').to_s).to eq ' #one·two·three'
     end
+
+    it 'does not match purely-numeric hashtags' do
+      expect(subject.match('hello #0123456')).to be_nil
+    end
   end
 
   describe '#to_param' do
-- 
cgit 


From bd87e6667975bc3b5bfaf3e1cdff97e041ed4c98 Mon Sep 17 00:00:00 2001
From: Eugen Rochko <eugen@zeonfederated.com>
Date: Sun, 21 Jul 2019 04:08:00 +0200
Subject: Remove WebSub subscriptions (#11303)

---
 app/models/concerns/account_associations.rb        |  3 -
 app/models/subscription.rb                         | 62 --------------------
 app/policies/subscription_policy.rb                |  7 ---
 app/services/suspend_account_service.rb            |  1 -
 .../scheduler/subscriptions_cleanup_scheduler.rb   |  4 +-
 config/sidekiq.yml                                 |  3 -
 .../20190715031050_drop_subscriptions.rb           | 11 ++++
 db/schema.rb                                       | 14 -----
 spec/fabricators/subscription_fabricator.rb        |  7 ---
 spec/models/subscription_spec.rb                   | 67 ----------------------
 spec/policies/subscription_policy_spec.rb          | 24 --------
 .../services/batched_remove_status_service_spec.rb |  3 -
 spec/services/remove_status_service_spec.rb        |  3 -
 spec/services/suspend_account_service_spec.rb      |  8 +--
 14 files changed, 14 insertions(+), 203 deletions(-)
 delete mode 100644 app/models/subscription.rb
 delete mode 100644 app/policies/subscription_policy.rb
 create mode 100644 db/post_migrate/20190715031050_drop_subscriptions.rb
 delete mode 100644 spec/fabricators/subscription_fabricator.rb
 delete mode 100644 spec/models/subscription_spec.rb
 delete mode 100644 spec/policies/subscription_policy_spec.rb

(limited to 'spec')

diff --git a/app/models/concerns/account_associations.rb b/app/models/concerns/account_associations.rb
index 0921e3252..1db7771c7 100644
--- a/app/models/concerns/account_associations.rb
+++ b/app/models/concerns/account_associations.rb
@@ -30,9 +30,6 @@ module AccountAssociations
     has_many :media_attachments, dependent: :destroy
     has_many :polls, dependent: :destroy
 
-    # PuSH subscriptions
-    has_many :subscriptions, dependent: :destroy
-
     # Report relationships
     has_many :reports, dependent: :destroy, inverse_of: :account
     has_many :targeted_reports, class_name: 'Report', foreign_key: :target_account_id, dependent: :destroy, inverse_of: :target_account
diff --git a/app/models/subscription.rb b/app/models/subscription.rb
deleted file mode 100644
index 79b81828d..000000000
--- a/app/models/subscription.rb
+++ /dev/null
@@ -1,62 +0,0 @@
-# frozen_string_literal: true
-# == Schema Information
-#
-# Table name: subscriptions
-#
-#  id                          :bigint(8)        not null, primary key
-#  callback_url                :string           default(""), not null
-#  secret                      :string
-#  expires_at                  :datetime
-#  confirmed                   :boolean          default(FALSE), not null
-#  created_at                  :datetime         not null
-#  updated_at                  :datetime         not null
-#  last_successful_delivery_at :datetime
-#  domain                      :string
-#  account_id                  :bigint(8)        not null
-#
-
-class Subscription < ApplicationRecord
-  MIN_EXPIRATION = 1.day.to_i
-  MAX_EXPIRATION = 30.days.to_i
-
-  belongs_to :account
-
-  validates :callback_url, presence: true
-  validates :callback_url, uniqueness: { scope: :account_id }
-
-  scope :confirmed, -> { where(confirmed: true) }
-  scope :future_expiration, -> { where(arel_table[:expires_at].gt(Time.now.utc)) }
-  scope :expired, -> { where(arel_table[:expires_at].lt(Time.now.utc)) }
-  scope :active, -> { confirmed.future_expiration }
-
-  def lease_seconds=(value)
-    self.expires_at = future_expiration(value)
-  end
-
-  def lease_seconds
-    (expires_at - Time.now.utc).to_i
-  end
-
-  def expired?
-    Time.now.utc > expires_at
-  end
-
-  before_validation :set_min_expiration
-
-  private
-
-  def future_expiration(value)
-    Time.now.utc + future_offset(value).seconds
-  end
-
-  def future_offset(seconds)
-    [
-      [MIN_EXPIRATION, seconds.to_i].max,
-      MAX_EXPIRATION,
-    ].min
-  end
-
-  def set_min_expiration
-    self.lease_seconds = 0 unless expires_at
-  end
-end
diff --git a/app/policies/subscription_policy.rb b/app/policies/subscription_policy.rb
deleted file mode 100644
index ac9a8a6c4..000000000
--- a/app/policies/subscription_policy.rb
+++ /dev/null
@@ -1,7 +0,0 @@
-# frozen_string_literal: true
-
-class SubscriptionPolicy < ApplicationPolicy
-  def index?
-    admin?
-  end
-end
diff --git a/app/services/suspend_account_service.rb b/app/services/suspend_account_service.rb
index 0ebe0b562..00cffcdfc 100644
--- a/app/services/suspend_account_service.rb
+++ b/app/services/suspend_account_service.rb
@@ -24,7 +24,6 @@ class SuspendAccountService < BaseService
     report_notes
     scheduled_statuses
     status_pins
-    subscriptions
   ).freeze
 
   ASSOCIATIONS_ON_DESTROY = %w(
diff --git a/app/workers/scheduler/subscriptions_cleanup_scheduler.rb b/app/workers/scheduler/subscriptions_cleanup_scheduler.rb
index 5fba120f6..75fe681a9 100644
--- a/app/workers/scheduler/subscriptions_cleanup_scheduler.rb
+++ b/app/workers/scheduler/subscriptions_cleanup_scheduler.rb
@@ -5,7 +5,5 @@ class Scheduler::SubscriptionsCleanupScheduler
 
   sidekiq_options unique: :until_executed, retry: 0
 
-  def perform
-    Subscription.expired.in_batches.delete_all
-  end
+  def perform; end
 end
diff --git a/config/sidekiq.yml b/config/sidekiq.yml
index 5c652792c..7f41b6607 100644
--- a/config/sidekiq.yml
+++ b/config/sidekiq.yml
@@ -21,9 +21,6 @@
   user_cleanup_scheduler:
     cron: '<%= Random.rand(0..59) %> <%= Random.rand(4..6) %> * * *'
     class: Scheduler::UserCleanupScheduler
-  subscriptions_cleanup_scheduler:
-    cron: '<%= Random.rand(0..59) %> <%= Random.rand(1..3) %> * * 0'
-    class: Scheduler::SubscriptionsCleanupScheduler
   ip_cleanup_scheduler:
     cron: '<%= Random.rand(0..59) %> <%= Random.rand(3..5) %> * * *'
     class: Scheduler::IpCleanupScheduler
diff --git a/db/post_migrate/20190715031050_drop_subscriptions.rb b/db/post_migrate/20190715031050_drop_subscriptions.rb
new file mode 100644
index 000000000..3719afe4a
--- /dev/null
+++ b/db/post_migrate/20190715031050_drop_subscriptions.rb
@@ -0,0 +1,11 @@
+# frozen_string_literal: true
+
+class DropSubscriptions < ActiveRecord::Migration[5.2]
+  def up
+    drop_table :subscriptions
+  end
+
+  def down
+    raise ActiveRecord::IrreversibleMigration
+  end
+end
diff --git a/db/schema.rb b/db/schema.rb
index a6a14827b..6319dd932 100644
--- a/db/schema.rb
+++ b/db/schema.rb
@@ -648,19 +648,6 @@ ActiveRecord::Schema.define(version: 2019_07_15_164535) do
     t.index ["tag_id", "status_id"], name: "index_statuses_tags_on_tag_id_and_status_id", unique: true
   end
 
-  create_table "subscriptions", force: :cascade do |t|
-    t.string "callback_url", default: "", null: false
-    t.string "secret"
-    t.datetime "expires_at"
-    t.boolean "confirmed", default: false, null: false
-    t.datetime "created_at", null: false
-    t.datetime "updated_at", null: false
-    t.datetime "last_successful_delivery_at"
-    t.string "domain"
-    t.bigint "account_id", null: false
-    t.index ["account_id", "callback_url"], name: "index_subscriptions_on_account_id_and_callback_url", unique: true
-  end
-
   create_table "tags", force: :cascade do |t|
     t.string "name", default: "", null: false
     t.datetime "created_at", null: false
@@ -821,7 +808,6 @@ ActiveRecord::Schema.define(version: 2019_07_15_164535) do
   add_foreign_key "statuses", "statuses", column: "reblog_of_id", on_delete: :cascade
   add_foreign_key "statuses_tags", "statuses", on_delete: :cascade
   add_foreign_key "statuses_tags", "tags", name: "fk_3081861e21", on_delete: :cascade
-  add_foreign_key "subscriptions", "accounts", name: "fk_9847d1cbb5", on_delete: :cascade
   add_foreign_key "tombstones", "accounts", on_delete: :cascade
   add_foreign_key "user_invite_requests", "users", on_delete: :cascade
   add_foreign_key "users", "accounts", name: "fk_50500f500d", on_delete: :cascade
diff --git a/spec/fabricators/subscription_fabricator.rb b/spec/fabricators/subscription_fabricator.rb
deleted file mode 100644
index 347dab5df..000000000
--- a/spec/fabricators/subscription_fabricator.rb
+++ /dev/null
@@ -1,7 +0,0 @@
-Fabricator(:subscription) do
-  account
-  callback_url "http://example.com/callback"
-  secret       "foobar"
-  expires_at   "2016-11-28 11:30:07"
-  confirmed    false
-end
diff --git a/spec/models/subscription_spec.rb b/spec/models/subscription_spec.rb
deleted file mode 100644
index b83979d13..000000000
--- a/spec/models/subscription_spec.rb
+++ /dev/null
@@ -1,67 +0,0 @@
-require 'rails_helper'
-
-RSpec.describe Subscription, type: :model do
-  let(:alice) { Fabricate(:account, username: 'alice') }
-
-  subject { Fabricate(:subscription, account: alice) }
-
-  describe '#expired?' do
-    it 'return true when expires_at is past' do
-      subject.expires_at = 2.days.ago
-      expect(subject.expired?).to be true
-    end
-
-    it 'return false when expires_at is future' do
-      subject.expires_at = 2.days.from_now
-      expect(subject.expired?).to be false
-    end
-  end
-
-  describe 'lease_seconds' do
-    it 'returns the time remaining until expiration' do
-      datetime = 1.day.from_now
-      subscription = Subscription.new(expires_at: datetime)
-      travel_to(datetime - 12.hours) do
-        expect(subscription.lease_seconds).to eq(12.hours)
-      end
-    end
-  end
-
-  describe 'lease_seconds=' do
-    it 'sets expires_at to min expiration when small value is provided' do
-      subscription = Subscription.new
-      datetime = 1.day.from_now
-      too_low = Subscription::MIN_EXPIRATION - 1000
-      travel_to(datetime) do
-        subscription.lease_seconds = too_low
-      end
-
-      expected = datetime + Subscription::MIN_EXPIRATION.seconds
-      expect(subscription.expires_at).to be_within(1.0).of(expected)
-    end
-
-    it 'sets expires_at to value when valid value is provided' do
-      subscription = Subscription.new
-      datetime = 1.day.from_now
-      valid = Subscription::MIN_EXPIRATION + 1000
-      travel_to(datetime) do
-        subscription.lease_seconds = valid
-      end
-
-      expected = datetime + valid.seconds
-      expect(subscription.expires_at).to be_within(1.0).of(expected)
-    end
-
-    it 'sets expires_at to max expiration when large value is provided' do
-      subscription = Subscription.new
-      datetime = 1.day.from_now
-      too_high = Subscription::MAX_EXPIRATION + 1000
-      travel_to(datetime) do
-        subscription.lease_seconds = too_high
-      end
-
-      expected = datetime + Subscription::MAX_EXPIRATION.seconds
-      expect(subscription.expires_at).to be_within(1.0).of(expected)
-    end
-  end
-end
diff --git a/spec/policies/subscription_policy_spec.rb b/spec/policies/subscription_policy_spec.rb
deleted file mode 100644
index 21d60c15f..000000000
--- a/spec/policies/subscription_policy_spec.rb
+++ /dev/null
@@ -1,24 +0,0 @@
-# frozen_string_literal: true
-
-require 'rails_helper'
-require 'pundit/rspec'
-
-RSpec.describe SubscriptionPolicy do
-  let(:subject) { described_class }
-  let(:admin)   { Fabricate(:user, admin: true).account }
-  let(:john)    { Fabricate(:user).account }
-
-  permissions :index? do
-    context 'admin?' do
-      it 'permits' do
-        expect(subject).to permit(admin, Subscription)
-      end
-    end
-
-    context '!admin?' do
-      it 'denies' do
-        expect(subject).to_not permit(john, Subscription)
-      end
-    end
-  end
-end
diff --git a/spec/services/batched_remove_status_service_spec.rb b/spec/services/batched_remove_status_service_spec.rb
index d52e7f484..f84256f18 100644
--- a/spec/services/batched_remove_status_service_spec.rb
+++ b/spec/services/batched_remove_status_service_spec.rb
@@ -14,11 +14,8 @@ RSpec.describe BatchedRemoveStatusService, type: :service do
   before do
     allow(Redis.current).to receive_messages(publish: nil)
 
-    stub_request(:post, 'http://example.com/push').to_return(status: 200, body: '', headers: {})
-    stub_request(:post, 'http://example.com/salmon').to_return(status: 200, body: '', headers: {})
     stub_request(:post, 'http://example.com/inbox').to_return(status: 200)
 
-    Fabricate(:subscription, account: alice, callback_url: 'http://example.com/push', confirmed: true, expires_at: 30.days.from_now)
     jeff.user.update(current_sign_in_at: Time.zone.now)
     jeff.follow!(alice)
     hank.follow!(alice)
diff --git a/spec/services/remove_status_service_spec.rb b/spec/services/remove_status_service_spec.rb
index 48191d47c..06676ec45 100644
--- a/spec/services/remove_status_service_spec.rb
+++ b/spec/services/remove_status_service_spec.rb
@@ -10,12 +10,9 @@ RSpec.describe RemoveStatusService, type: :service do
   let!(:bill)   { Fabricate(:account, username: 'bill', protocol: :activitypub, domain: 'example2.com', inbox_url: 'http://example2.com/inbox') }
 
   before do
-    stub_request(:post, 'http://example.com/push').to_return(status: 200, body: '', headers: {})
-    stub_request(:post, 'http://example.com/salmon').to_return(status: 200, body: '', headers: {})
     stub_request(:post, 'http://example.com/inbox').to_return(status: 200)
     stub_request(:post, 'http://example2.com/inbox').to_return(status: 200)
 
-    Fabricate(:subscription, account: alice, callback_url: 'http://example.com/push', confirmed: true, expires_at: 30.days.from_now)
     jeff.follow!(alice)
     hank.follow!(alice)
 
diff --git a/spec/services/suspend_account_service_spec.rb b/spec/services/suspend_account_service_spec.rb
index 896ac17a3..eebbbc12a 100644
--- a/spec/services/suspend_account_service_spec.rb
+++ b/spec/services/suspend_account_service_spec.rb
@@ -18,7 +18,6 @@ RSpec.describe SuspendAccountService, type: :service do
     let!(:favourite) { Fabricate(:favourite, account: account) }
     let!(:active_relationship) { Fabricate(:follow, account: account) }
     let!(:passive_relationship) { Fabricate(:follow, target_account: account) }
-    let!(:subscription) { Fabricate(:subscription, account: account) }
     let!(:remote_alice) { Fabricate(:account, inbox_url: 'https://alice.com/inbox', protocol: :activitypub) }
     let!(:remote_bob) { Fabricate(:account, inbox_url: 'https://bob.com/inbox', protocol: :activitypub) }
 
@@ -31,9 +30,8 @@ RSpec.describe SuspendAccountService, type: :service do
           account.favourites,
           account.active_relationships,
           account.passive_relationships,
-          account.subscriptions
         ].map(&:count)
-      }.from([1, 1, 1, 1, 1, 1, 1]).to([0, 0, 0, 0, 0, 0, 0])
+      }.from([1, 1, 1, 1, 1, 1]).to([0, 0, 0, 0, 0, 0])
     end
 
     it 'sends a delete actor activity to all known inboxes' do
@@ -62,7 +60,6 @@ RSpec.describe SuspendAccountService, type: :service do
     let!(:favourite) { Fabricate(:favourite, account: remote_bob) }
     let!(:active_relationship) { Fabricate(:follow, account: remote_bob, target_account: account) }
     let!(:passive_relationship) { Fabricate(:follow, target_account: remote_bob) }
-    let!(:subscription) { Fabricate(:subscription, account: remote_bob) }
 
     it 'deletes associated records' do
       is_expected.to change {
@@ -73,9 +70,8 @@ RSpec.describe SuspendAccountService, type: :service do
           remote_bob.favourites,
           remote_bob.active_relationships,
           remote_bob.passive_relationships,
-          remote_bob.subscriptions
         ].map(&:count)
-      }.from([1, 1, 1, 1, 1, 1, 1]).to([0, 0, 0, 0, 0, 0, 0])
+      }.from([1, 1, 1, 1, 1, 1]).to([0, 0, 0, 0, 0, 0])
     end
 
     it 'sends a reject follow to follwer inboxes' do
-- 
cgit 


From bd1545de5ee9655130e5357bb9cb6449520a6292 Mon Sep 17 00:00:00 2001
From: Eugen Rochko <eugen@zeonfederated.com>
Date: Sun, 21 Jul 2019 18:08:02 +0200
Subject: Change locale detection to run once per session (#8657)

Fix #6462
---
 app/controllers/application_controller.rb   |  6 +-----
 app/controllers/concerns/localized.rb       | 13 ++++++++-----
 config/application.rb                       |  3 +++
 spec/controllers/concerns/localized_spec.rb | 16 +++++-----------
 4 files changed, 17 insertions(+), 21 deletions(-)

(limited to 'spec')

diff --git a/app/controllers/application_controller.rb b/app/controllers/application_controller.rb
index 51e9764d4..5863fe168 100644
--- a/app/controllers/application_controller.rb
+++ b/app/controllers/application_controller.rb
@@ -138,11 +138,7 @@ class ApplicationController < ActionController::Base
   def respond_with_error(code)
     respond_to do |format|
       format.any  { head code }
-
-      format.html do
-        set_locale
-        render "errors/#{code}", layout: 'error', status: code
-      end
+      format.html { render "errors/#{code}", layout: 'error', status: code }
     end
   end
 
diff --git a/app/controllers/concerns/localized.rb b/app/controllers/concerns/localized.rb
index 145549bcd..b43859d9d 100644
--- a/app/controllers/concerns/localized.rb
+++ b/app/controllers/concerns/localized.rb
@@ -4,16 +4,19 @@ module Localized
   extend ActiveSupport::Concern
 
   included do
-    before_action :set_locale
+    around_action :set_locale
   end
 
   private
 
   def set_locale
-    I18n.locale = default_locale
-    I18n.locale = current_user.locale if user_signed_in?
-  rescue I18n::InvalidLocale
-    I18n.locale = default_locale
+    locale   = current_user.locale if respond_to?(:user_signed_in?) && user_signed_in?
+    locale ||= session[:locale] ||= default_locale
+    locale   = default_locale unless I18n.available_locales.include?(locale.to_sym)
+
+    I18n.with_locale(locale) do
+      yield
+    end
   end
 
   def default_locale
diff --git a/config/application.rb b/config/application.rb
index 4534ede49..f49deffbb 100644
--- a/config/application.rb
+++ b/config/application.rb
@@ -114,6 +114,9 @@ module Mastodon
       Doorkeeper::AuthorizationsController.layout 'modal'
       Doorkeeper::AuthorizedApplicationsController.layout 'admin'
       Doorkeeper::Application.send :include, ApplicationExtension
+      Devise::FailureApp.send :include, AbstractController::Callbacks
+      Devise::FailureApp.send :include, HttpAcceptLanguage::EasyAccess
+      Devise::FailureApp.send :include, Localized
     end
   end
 end
diff --git a/spec/controllers/concerns/localized_spec.rb b/spec/controllers/concerns/localized_spec.rb
index 76c3de118..7635d10e1 100644
--- a/spec/controllers/concerns/localized_spec.rb
+++ b/spec/controllers/concerns/localized_spec.rb
@@ -7,16 +7,10 @@ describe ApplicationController, type: :controller do
     include Localized
 
     def success
-      head 200
+      render plain: I18n.locale, status: 200
     end
   end
 
-  around do |example|
-    current_locale = I18n.locale
-    example.run
-    I18n.locale = current_locale
-  end
-
   before do
     routes.draw { get 'success' => 'anonymous#success' }
   end
@@ -25,19 +19,19 @@ describe ApplicationController, type: :controller do
     it 'sets available and preferred language' do
       request.headers['Accept-Language'] = 'ca-ES, fa'
       get 'success'
-      expect(I18n.locale).to eq :fa
+      expect(response.body).to eq 'fa'
     end
 
     it 'sets available and compatible language if none of available languages are preferred' do
       request.headers['Accept-Language'] = 'fa-IR'
       get 'success'
-      expect(I18n.locale).to eq :fa
+      expect(response.body).to eq 'fa'
     end
 
     it 'sets default locale if none of available languages are compatible' do
       request.headers['Accept-Language'] = ''
       get 'success'
-      expect(I18n.locale).to eq :en
+      expect(response.body).to eq 'en'
     end
   end
 
@@ -48,7 +42,7 @@ describe ApplicationController, type: :controller do
       sign_in(user)
       get 'success'
 
-      expect(I18n.locale).to eq :ca
+      expect(response.body).to eq 'ca'
     end
   end
 
-- 
cgit 


From 7de8c51873b51d8450f7a6597a43d454964d0407 Mon Sep 17 00:00:00 2001
From: ThibG <thib@sitedethib.com>
Date: Sun, 21 Jul 2019 18:10:40 +0200
Subject: Play animated custom emoji on hover (#11348)

* Play animated custom emoji on hover in status

* Play animated custom emoji on hover in display names

* Play animated custom emoji on hover in bios/bio fields

* Add support for animation on hover on public pages emojis too

* Fix tests

* Code style cleanup
---
 app/javascript/mastodon/components/display_name.js | 44 +++++++++++++++++++++-
 .../mastodon/components/status_content.js          | 32 ++++++++++++++++
 .../mastodon/features/account/components/header.js | 43 ++++++++++++++++++++-
 app/javascript/mastodon/features/emoji/emoji.js    |  2 +-
 app/javascript/packs/public.js                     |  9 +++++
 app/lib/formatter.rb                               | 15 +++++---
 spec/lib/formatter_spec.rb                         | 26 ++++++-------
 7 files changed, 149 insertions(+), 22 deletions(-)

(limited to 'spec')

diff --git a/app/javascript/mastodon/components/display_name.js b/app/javascript/mastodon/components/display_name.js
index 6b9dd6f81..70ef82789 100644
--- a/app/javascript/mastodon/components/display_name.js
+++ b/app/javascript/mastodon/components/display_name.js
@@ -1,6 +1,7 @@
 import React from 'react';
 import ImmutablePropTypes from 'react-immutable-proptypes';
 import PropTypes from 'prop-types';
+import { autoPlayGif } from 'mastodon/initial_state';
 
 export default class DisplayName extends React.PureComponent {
 
@@ -10,6 +11,47 @@ export default class DisplayName extends React.PureComponent {
     localDomain: PropTypes.string,
   };
 
+  _updateEmojis () {
+    const node = this.node;
+
+    if (!node || autoPlayGif) {
+      return;
+    }
+
+    const emojis = node.querySelectorAll('.custom-emoji');
+
+    for (var i = 0; i < emojis.length; i++) {
+      let emoji = emojis[i];
+      if (emoji.classList.contains('status-emoji')) {
+        continue;
+      }
+      emoji.classList.add('status-emoji');
+
+      emoji.addEventListener('mouseenter', this.handleEmojiMouseEnter, false);
+      emoji.addEventListener('mouseleave', this.handleEmojiMouseLeave, false);
+    }
+  }
+
+  componentDidMount () {
+    this._updateEmojis();
+  }
+
+  componentDidUpdate () {
+    this._updateEmojis();
+  }
+
+  handleEmojiMouseEnter = ({ target }) => {
+    target.src = target.getAttribute('data-original');
+  }
+
+  handleEmojiMouseLeave = ({ target }) => {
+    target.src = target.getAttribute('data-static');
+  }
+
+  setRef = (c) => {
+    this.node = c;
+  }
+
   render () {
     const { others, localDomain } = this.props;
 
@@ -39,7 +81,7 @@ export default class DisplayName extends React.PureComponent {
     }
 
     return (
-      <span className='display-name'>
+      <span className='display-name' ref={this.setRef}>
         {displayName} {suffix}
       </span>
     );
diff --git a/app/javascript/mastodon/components/status_content.js b/app/javascript/mastodon/components/status_content.js
index 06f5b4aad..8a05415af 100644
--- a/app/javascript/mastodon/components/status_content.js
+++ b/app/javascript/mastodon/components/status_content.js
@@ -7,6 +7,7 @@ import Permalink from './permalink';
 import classnames from 'classnames';
 import PollContainer from 'mastodon/containers/poll_container';
 import Icon from 'mastodon/components/icon';
+import { autoPlayGif } from 'mastodon/initial_state';
 
 const MAX_HEIGHT = 642; // 20px * 32 (+ 2px padding at the top)
 
@@ -71,12 +72,35 @@ export default class StatusContent extends React.PureComponent {
     }
   }
 
+  _updateStatusEmojis () {
+    const node = this.node;
+
+    if (!node || autoPlayGif) {
+      return;
+    }
+
+    const emojis = node.querySelectorAll('.custom-emoji');
+
+    for (var i = 0; i < emojis.length; i++) {
+      let emoji = emojis[i];
+      if (emoji.classList.contains('status-emoji')) {
+        continue;
+      }
+      emoji.classList.add('status-emoji');
+
+      emoji.addEventListener('mouseenter', this.handleEmojiMouseEnter, false);
+      emoji.addEventListener('mouseleave', this.handleEmojiMouseLeave, false);
+    }
+  }
+
   componentDidMount () {
     this._updateStatusLinks();
+    this._updateStatusEmojis();
   }
 
   componentDidUpdate () {
     this._updateStatusLinks();
+    this._updateStatusEmojis();
   }
 
   onMentionClick = (mention, e) => {
@@ -95,6 +119,14 @@ export default class StatusContent extends React.PureComponent {
     }
   }
 
+  handleEmojiMouseEnter = ({ target }) => {
+    target.src = target.getAttribute('data-original');
+  }
+
+  handleEmojiMouseLeave = ({ target }) => {
+    target.src = target.getAttribute('data-static');
+  }
+
   handleMouseDown = (e) => {
     this.startXY = [e.clientX, e.clientY];
   }
diff --git a/app/javascript/mastodon/features/account/components/header.js b/app/javascript/mastodon/features/account/components/header.js
index e5b60e33e..cab67c607 100644
--- a/app/javascript/mastodon/features/account/components/header.js
+++ b/app/javascript/mastodon/features/account/components/header.js
@@ -79,6 +79,47 @@ class Header extends ImmutablePureComponent {
     return !location.pathname.match(/\/(followers|following)\/?$/);
   }
 
+  _updateEmojis () {
+    const node = this.node;
+
+    if (!node || autoPlayGif) {
+      return;
+    }
+
+    const emojis = node.querySelectorAll('.custom-emoji');
+
+    for (var i = 0; i < emojis.length; i++) {
+      let emoji = emojis[i];
+      if (emoji.classList.contains('status-emoji')) {
+        continue;
+      }
+      emoji.classList.add('status-emoji');
+
+      emoji.addEventListener('mouseenter', this.handleEmojiMouseEnter, false);
+      emoji.addEventListener('mouseleave', this.handleEmojiMouseLeave, false);
+    }
+  }
+
+  componentDidMount () {
+    this._updateEmojis();
+  }
+
+  componentDidUpdate () {
+    this._updateEmojis();
+  }
+
+  handleEmojiMouseEnter = ({ target }) => {
+    target.src = target.getAttribute('data-original');
+  }
+
+  handleEmojiMouseLeave = ({ target }) => {
+    target.src = target.getAttribute('data-static');
+  }
+
+  setRef = (c) => {
+    this.node = c;
+  }
+
   render () {
     const { account, intl, domain, identity_proofs } = this.props;
 
@@ -200,7 +241,7 @@ class Header extends ImmutablePureComponent {
     const acct            = account.get('acct').indexOf('@') === -1 && domain ? `${account.get('acct')}@${domain}` : account.get('acct');
 
     return (
-      <div className={classNames('account__header', { inactive: !!account.get('moved') })}>
+      <div className={classNames('account__header', { inactive: !!account.get('moved') })} ref={this.setRef}>
         <div className='account__header__image'>
           <div className='account__header__info'>
             {info}
diff --git a/app/javascript/mastodon/features/emoji/emoji.js b/app/javascript/mastodon/features/emoji/emoji.js
index 01b5a6664..359bb7ffd 100644
--- a/app/javascript/mastodon/features/emoji/emoji.js
+++ b/app/javascript/mastodon/features/emoji/emoji.js
@@ -29,7 +29,7 @@ const emojify = (str, customEmojis = {}) => {
         // if you want additional emoji handler, add statements below which set replacement and return true.
         if (shortname in customEmojis) {
           const filename = autoPlayGif ? customEmojis[shortname].url : customEmojis[shortname].static_url;
-          replacement = `<img draggable="false" class="emojione" alt="${shortname}" title="${shortname}" src="${filename}" />`;
+          replacement = `<img draggable="false" class="emojione custom-emoji" alt="${shortname}" title="${shortname}" src="${filename}" data-original="${customEmojis[shortname].url}" data-static="${customEmojis[shortname].static_url}" />`;
           return true;
         }
         return false;
diff --git a/app/javascript/packs/public.js b/app/javascript/packs/public.js
index 0c60d828e..b58622a8d 100644
--- a/app/javascript/packs/public.js
+++ b/app/javascript/packs/public.js
@@ -44,6 +44,12 @@ function main() {
     }
   };
 
+  const getEmojiAnimationHandler = (swapTo) => {
+    return ({ target }) => {
+      target.src = target.getAttribute(swapTo);
+    };
+  };
+
   ready(() => {
     const locale = document.documentElement.lang;
 
@@ -108,6 +114,9 @@ function main() {
     if (parallaxComponents.length > 0 ) {
       new Rellax('.parallax', { speed: -1 });
     }
+
+    delegate(document, '.custom-emoji', 'mouseover', getEmojiAnimationHandler('data-original'));
+    delegate(document, '.custom-emoji', 'mouseout', getEmojiAnimationHandler('data-static'));
   });
 
   delegate(document, '.webapp-btn', 'click', ({ target, button }) => {
diff --git a/app/lib/formatter.rb b/app/lib/formatter.rb
index 6c1239963..65059efa0 100644
--- a/app/lib/formatter.rb
+++ b/app/lib/formatter.rb
@@ -137,11 +137,7 @@ class Formatter
   def encode_custom_emojis(html, emojis, animate = false)
     return html if emojis.empty?
 
-    emoji_map = if animate
-                  emojis.each_with_object({}) { |e, h| h[e.shortcode] = full_asset_url(e.image.url) }
-                else
-                  emojis.each_with_object({}) { |e, h| h[e.shortcode] = full_asset_url(e.image.url(:static)) }
-                end
+    emoji_map = emojis.each_with_object({}) { |e, h| h[e.shortcode] = [full_asset_url(e.image.url), full_asset_url(e.image.url(:static))] }
 
     i                     = -1
     tag_open_index        = nil
@@ -157,7 +153,14 @@ class Formatter
         emoji     = emoji_map[shortcode]
 
         if emoji
-          replacement = "<img draggable=\"false\" class=\"emojione\" alt=\":#{encode(shortcode)}:\" title=\":#{encode(shortcode)}:\" src=\"#{encode(emoji)}\" />"
+          original_url, static_url = emoji
+          replacement = begin
+            if animate
+              "<img draggable=\"false\" class=\"emojione\" alt=\":#{encode(shortcode)}:\" title=\":#{encode(shortcode)}:\" src=\"#{encode(original_url)}\" />"
+            else
+              "<img draggable=\"false\" class=\"emojione custom-emoji\" alt=\":#{encode(shortcode)}:\" title=\":#{encode(shortcode)}:\" src=\"#{encode(static_url)}\" data-original=\"#{original_url}\" data-static=\"#{static_url}\" />"
+            end
+          end
           before_html = shortname_start_index.positive? ? html[0..shortname_start_index - 1] : ''
           html        = before_html + replacement + html[i + 1..-1]
           i          += replacement.size - (shortcode.size + 2) - 1
diff --git a/spec/lib/formatter_spec.rb b/spec/lib/formatter_spec.rb
index 96d2fc7e0..b8108a247 100644
--- a/spec/lib/formatter_spec.rb
+++ b/spec/lib/formatter_spec.rb
@@ -261,7 +261,7 @@ RSpec.describe Formatter do
       let(:text) { ':coolcat: Beep boop' }
 
       it 'converts the shortcode to an image tag' do
-        is_expected.to match(/<img draggable="false" class="emojione" alt=":coolcat:"/)
+        is_expected.to match(/<img draggable="false" class="emojione custom-emoji" alt=":coolcat:"/)
       end
     end
   end
@@ -330,7 +330,7 @@ RSpec.describe Formatter do
           let(:text) { ':coolcat: Beep boop' }
 
           it 'converts the shortcode to an image tag' do
-            is_expected.to match(/<p><img draggable="false" class="emojione" alt=":coolcat:"/)
+            is_expected.to match(/<p><img draggable="false" class="emojione custom-emoji" alt=":coolcat:"/)
           end
         end
 
@@ -338,7 +338,7 @@ RSpec.describe Formatter do
           let(:text) { 'Beep :coolcat: boop' }
 
           it 'converts the shortcode to an image tag' do
-            is_expected.to match(/Beep <img draggable="false" class="emojione" alt=":coolcat:"/)
+            is_expected.to match(/Beep <img draggable="false" class="emojione custom-emoji" alt=":coolcat:"/)
           end
         end
 
@@ -354,7 +354,7 @@ RSpec.describe Formatter do
           let(:text) { 'Beep boop :coolcat:' }
 
           it 'converts the shortcode to an image tag' do
-            is_expected.to match(/boop <img draggable="false" class="emojione" alt=":coolcat:"/)
+            is_expected.to match(/boop <img draggable="false" class="emojione custom-emoji" alt=":coolcat:"/)
           end
         end
       end
@@ -377,7 +377,7 @@ RSpec.describe Formatter do
           let(:text) { '<p>:coolcat: Beep boop<br />' }
 
           it 'converts the shortcode to an image tag' do
-            is_expected.to match(/<p><img draggable="false" class="emojione" alt=":coolcat:"/)
+            is_expected.to match(/<p><img draggable="false" class="emojione custom-emoji" alt=":coolcat:"/)
           end
         end
 
@@ -385,7 +385,7 @@ RSpec.describe Formatter do
           let(:text) { '<p>Beep :coolcat: boop</p>' }
 
           it 'converts the shortcode to an image tag' do
-            is_expected.to match(/Beep <img draggable="false" class="emojione" alt=":coolcat:"/)
+            is_expected.to match(/Beep <img draggable="false" class="emojione custom-emoji" alt=":coolcat:"/)
           end
         end
 
@@ -401,7 +401,7 @@ RSpec.describe Formatter do
           let(:text) { '<p>Beep boop<br />:coolcat:</p>' }
 
           it 'converts the shortcode to an image tag' do
-            is_expected.to match(/<br><img draggable="false" class="emojione" alt=":coolcat:"/)
+            is_expected.to match(/<br><img draggable="false" class="emojione custom-emoji" alt=":coolcat:"/)
           end
         end
       end
@@ -500,7 +500,7 @@ RSpec.describe Formatter do
           let(:text) { ':coolcat: Beep boop' }
 
           it 'converts the shortcode to an image tag' do
-            is_expected.to match(/<p><img draggable="false" class="emojione" alt=":coolcat:"/)
+            is_expected.to match(/<p><img draggable="false" class="emojione custom-emoji" alt=":coolcat:"/)
           end
         end
 
@@ -508,7 +508,7 @@ RSpec.describe Formatter do
           let(:text) { 'Beep :coolcat: boop' }
 
           it 'converts the shortcode to an image tag' do
-            is_expected.to match(/Beep <img draggable="false" class="emojione" alt=":coolcat:"/)
+            is_expected.to match(/Beep <img draggable="false" class="emojione custom-emoji" alt=":coolcat:"/)
           end
         end
 
@@ -524,7 +524,7 @@ RSpec.describe Formatter do
           let(:text) { 'Beep boop :coolcat:' }
 
           it 'converts the shortcode to an image tag' do
-            is_expected.to match(/boop <img draggable="false" class="emojione" alt=":coolcat:"/)
+            is_expected.to match(/boop <img draggable="false" class="emojione custom-emoji" alt=":coolcat:"/)
           end
         end
       end
@@ -551,7 +551,7 @@ RSpec.describe Formatter do
           let(:text) { '<p>:coolcat: Beep boop<br />' }
 
           it 'converts shortcode to image tag' do
-            is_expected.to match(/<p><img draggable="false" class="emojione" alt=":coolcat:"/)
+            is_expected.to match(/<p><img draggable="false" class="emojione custom-emoji" alt=":coolcat:"/)
           end
         end
 
@@ -559,7 +559,7 @@ RSpec.describe Formatter do
           let(:text) { '<p>Beep :coolcat: boop</p>' }
 
           it 'converts shortcode to image tag' do
-            is_expected.to match(/Beep <img draggable="false" class="emojione" alt=":coolcat:"/)
+            is_expected.to match(/Beep <img draggable="false" class="emojione custom-emoji" alt=":coolcat:"/)
           end
         end
 
@@ -575,7 +575,7 @@ RSpec.describe Formatter do
           let(:text) { '<p>Beep boop<br />:coolcat:</p>' }
 
           it 'converts shortcode to image tag' do
-            is_expected.to match(/<br><img draggable="false" class="emojione" alt=":coolcat:"/)
+            is_expected.to match(/<br><img draggable="false" class="emojione custom-emoji" alt=":coolcat:"/)
           end
         end
       end
-- 
cgit 


From 964ae8eee593687f922c873fa7b378bb6e3e39bb Mon Sep 17 00:00:00 2001
From: Eugen Rochko <eugen@zeonfederated.com>
Date: Mon, 22 Jul 2019 10:48:50 +0200
Subject: Change unconfirmed user login behaviour (#11375)

Allow access to account settings, 2FA, authorized applications, and
account deletions to unconfirmed and pending users, as well as
users who had their accounts disabled. Suspended users cannot update
their e-mail or password or delete their account.

Display account status on account settings page, for example, when
an account is frozen, limited, unconfirmed or pending review.

After sign up, login users straight away and show a simple page that
tells them the status of their account with links to account settings
and logout, to reduce onboarding friction and allow users to correct
wrongly typed e-mail addresses.

Move the final sign-up step of SSO integrations to be the same
as above to reduce code duplication.
---
 app/controllers/about_controller.rb                |  2 +-
 app/controllers/api/base_controller.rb             |  2 +-
 app/controllers/application_controller.rb          |  6 +--
 app/controllers/auth/confirmations_controller.rb   | 21 +-------
 .../auth/omniauth_callbacks_controller.rb          |  2 +-
 app/controllers/auth/registrations_controller.rb   |  9 +++-
 app/controllers/auth/sessions_controller.rb        |  4 +-
 app/controllers/auth/setup_controller.rb           | 58 ++++++++++++++++++++++
 .../oauth/authorized_applications_controller.rb    |  2 +
 app/controllers/settings/deletes_controller.rb     |  7 +++
 app/controllers/settings/sessions_controller.rb    |  2 +
 .../confirmations_controller.rb                    |  2 +
 .../recovery_codes_controller.rb                   |  2 +
 .../two_factor_authentications_controller.rb       |  2 +
 app/javascript/styles/mastodon/admin.scss          | 58 +++++++++++++---------
 app/javascript/styles/mastodon/forms.scss          |  7 +++
 app/models/concerns/omniauthable.rb                |  2 +-
 app/models/user.rb                                 |  6 ++-
 .../auth/confirmations/finish_signup.html.haml     | 15 ------
 app/views/auth/registrations/_sessions.html.haml   |  4 +-
 app/views/auth/registrations/_status.html.haml     | 16 ++++++
 app/views/auth/registrations/edit.html.haml        | 35 +++++++------
 app/views/auth/setup/show.html.haml                | 23 +++++++++
 .../oauth/authorized_applications/index.html.haml  |  2 +-
 config/locales/en.yml                              |  9 +++-
 config/routes.rb                                   |  5 +-
 db/seeds.rb                                        |  2 +-
 spec/controllers/api/base_controller_spec.rb       | 42 +++++++++++++++-
 spec/controllers/application_controller_spec.rb    |  4 +-
 .../auth/confirmations_controller_spec.rb          | 41 ---------------
 .../auth/registrations_controller_spec.rb          | 25 +++++++---
 spec/controllers/auth/sessions_controller_spec.rb  |  4 +-
 .../settings/deletes_controller_spec.rb            | 17 +++++++
 spec/features/log_in_spec.rb                       |  4 +-
 spec/models/user_spec.rb                           |  4 +-
 35 files changed, 298 insertions(+), 148 deletions(-)
 create mode 100644 app/controllers/auth/setup_controller.rb
 delete mode 100644 app/views/auth/confirmations/finish_signup.html.haml
 create mode 100644 app/views/auth/registrations/_status.html.haml
 create mode 100644 app/views/auth/setup/show.html.haml

(limited to 'spec')

diff --git a/app/controllers/about_controller.rb b/app/controllers/about_controller.rb
index 33bac9bbc..31cf17710 100644
--- a/app/controllers/about_controller.rb
+++ b/app/controllers/about_controller.rb
@@ -7,7 +7,7 @@ class AboutController < ApplicationController
   before_action :set_instance_presenter
   before_action :set_expires_in
 
-  skip_before_action :check_user_permissions, only: [:more, :terms]
+  skip_before_action :require_functional!, only: [:more, :terms]
 
   def show; end
 
diff --git a/app/controllers/api/base_controller.rb b/app/controllers/api/base_controller.rb
index eca558f42..6f33a1ea9 100644
--- a/app/controllers/api/base_controller.rb
+++ b/app/controllers/api/base_controller.rb
@@ -7,7 +7,7 @@ class Api::BaseController < ApplicationController
   include RateLimitHeaders
 
   skip_before_action :store_current_location
-  skip_before_action :check_user_permissions
+  skip_before_action :require_functional!
 
   before_action :set_cache_headers
 
diff --git a/app/controllers/application_controller.rb b/app/controllers/application_controller.rb
index b8a1faf77..41ce1a0ca 100644
--- a/app/controllers/application_controller.rb
+++ b/app/controllers/application_controller.rb
@@ -25,7 +25,7 @@ class ApplicationController < ActionController::Base
   rescue_from Mastodon::NotPermittedError, with: :forbidden
 
   before_action :store_current_location, except: :raise_not_found, unless: :devise_controller?
-  before_action :check_user_permissions, if: :user_signed_in?
+  before_action :require_functional!, if: :user_signed_in?
 
   def raise_not_found
     raise ActionController::RoutingError, "No route matches #{params[:unmatched_route]}"
@@ -57,8 +57,8 @@ class ApplicationController < ActionController::Base
     forbidden unless current_user&.staff?
   end
 
-  def check_user_permissions
-    forbidden if current_user.disabled? || current_user.account.suspended?
+  def require_functional!
+    redirect_to edit_user_registration_path unless current_user.functional?
   end
 
   def after_sign_out_path_for(_resource_or_scope)
diff --git a/app/controllers/auth/confirmations_controller.rb b/app/controllers/auth/confirmations_controller.rb
index c28c7471c..0d7c6e7c2 100644
--- a/app/controllers/auth/confirmations_controller.rb
+++ b/app/controllers/auth/confirmations_controller.rb
@@ -4,34 +4,15 @@ class Auth::ConfirmationsController < Devise::ConfirmationsController
   layout 'auth'
 
   before_action :set_body_classes
-  before_action :set_user, only: [:finish_signup]
 
-  def finish_signup
-    return unless request.patch? && params[:user]
-
-    if @user.update(user_params)
-      @user.skip_reconfirmation!
-      bypass_sign_in(@user)
-      redirect_to root_path, notice: I18n.t('devise.confirmations.send_instructions')
-    else
-      @show_errors = true
-    end
-  end
+  skip_before_action :require_functional!
 
   private
 
-  def set_user
-    @user = current_user
-  end
-
   def set_body_classes
     @body_classes = 'lighter'
   end
 
-  def user_params
-    params.require(:user).permit(:email)
-  end
-
   def after_confirmation_path_for(_resource_name, user)
     if user.created_by_application && truthy_param?(:redirect_to_app)
       user.created_by_application.redirect_uri
diff --git a/app/controllers/auth/omniauth_callbacks_controller.rb b/app/controllers/auth/omniauth_callbacks_controller.rb
index bbf63bed3..682c77016 100644
--- a/app/controllers/auth/omniauth_callbacks_controller.rb
+++ b/app/controllers/auth/omniauth_callbacks_controller.rb
@@ -27,7 +27,7 @@ class Auth::OmniauthCallbacksController < Devise::OmniauthCallbacksController
     if resource.email_verified?
       root_path
     else
-      finish_signup_path
+      auth_setup_path(missing_email: '1')
     end
   end
 end
diff --git a/app/controllers/auth/registrations_controller.rb b/app/controllers/auth/registrations_controller.rb
index 83797cf1f..019caf9c1 100644
--- a/app/controllers/auth/registrations_controller.rb
+++ b/app/controllers/auth/registrations_controller.rb
@@ -9,6 +9,9 @@ class Auth::RegistrationsController < Devise::RegistrationsController
   before_action :set_sessions, only: [:edit, :update]
   before_action :set_instance_presenter, only: [:new, :create, :update]
   before_action :set_body_classes, only: [:new, :create, :edit, :update]
+  before_action :require_not_suspended!, only: [:update]
+
+  skip_before_action :require_functional!, only: [:edit, :update]
 
   def new
     super(&:build_invite_request)
@@ -43,7 +46,7 @@ class Auth::RegistrationsController < Devise::RegistrationsController
   end
 
   def after_sign_up_path_for(_resource)
-    new_user_session_path
+    auth_setup_path
   end
 
   def after_sign_in_path_for(_resource)
@@ -102,4 +105,8 @@ class Auth::RegistrationsController < Devise::RegistrationsController
   def set_sessions
     @sessions = current_user.session_activations
   end
+
+  def require_not_suspended!
+    forbidden if current_account.suspended?
+  end
 end
diff --git a/app/controllers/auth/sessions_controller.rb b/app/controllers/auth/sessions_controller.rb
index fb8615c31..7e6dbf19e 100644
--- a/app/controllers/auth/sessions_controller.rb
+++ b/app/controllers/auth/sessions_controller.rb
@@ -6,8 +6,10 @@ class Auth::SessionsController < Devise::SessionsController
   layout 'auth'
 
   skip_before_action :require_no_authentication, only: [:create]
-  skip_before_action :check_user_permissions, only: [:destroy]
+  skip_before_action :require_functional!
+
   prepend_before_action :authenticate_with_two_factor, if: :two_factor_enabled?, only: [:create]
+
   before_action :set_instance_presenter, only: [:new]
   before_action :set_body_classes
 
diff --git a/app/controllers/auth/setup_controller.rb b/app/controllers/auth/setup_controller.rb
new file mode 100644
index 000000000..46c5f2958
--- /dev/null
+++ b/app/controllers/auth/setup_controller.rb
@@ -0,0 +1,58 @@
+# frozen_string_literal: true
+
+class Auth::SetupController < ApplicationController
+  layout 'auth'
+
+  before_action :authenticate_user!
+  before_action :require_unconfirmed_or_pending!
+  before_action :set_body_classes
+  before_action :set_user
+
+  skip_before_action :require_functional!
+
+  def show
+    flash.now[:notice] = begin
+      if @user.pending?
+        I18n.t('devise.registrations.signed_up_but_pending')
+      else
+        I18n.t('devise.registrations.signed_up_but_unconfirmed')
+      end
+    end
+  end
+
+  def update
+    # This allows updating the e-mail without entering a password as is required
+    # on the account settings page; however, we only allow this for accounts
+    # that were not confirmed yet
+
+    if @user.update(user_params)
+      redirect_to auth_setup_path, notice: I18n.t('devise.confirmations.send_instructions')
+    else
+      render :show
+    end
+  end
+
+  helper_method :missing_email?
+
+  private
+
+  def require_unconfirmed_or_pending!
+    redirect_to root_path if current_user.confirmed? && current_user.approved?
+  end
+
+  def set_user
+    @user = current_user
+  end
+
+  def set_body_classes
+    @body_classes = 'lighter'
+  end
+
+  def user_params
+    params.require(:user).permit(:email)
+  end
+
+  def missing_email?
+    truthy_param?(:missing_email)
+  end
+end
diff --git a/app/controllers/oauth/authorized_applications_controller.rb b/app/controllers/oauth/authorized_applications_controller.rb
index f3d235366..fb8389034 100644
--- a/app/controllers/oauth/authorized_applications_controller.rb
+++ b/app/controllers/oauth/authorized_applications_controller.rb
@@ -7,6 +7,8 @@ class Oauth::AuthorizedApplicationsController < Doorkeeper::AuthorizedApplicatio
   before_action :authenticate_resource_owner!
   before_action :set_body_classes
 
+  skip_before_action :require_functional!
+
   include Localized
 
   def destroy
diff --git a/app/controllers/settings/deletes_controller.rb b/app/controllers/settings/deletes_controller.rb
index dd19aadf6..97fe4d328 100644
--- a/app/controllers/settings/deletes_controller.rb
+++ b/app/controllers/settings/deletes_controller.rb
@@ -5,6 +5,9 @@ class Settings::DeletesController < Settings::BaseController
 
   before_action :check_enabled_deletion
   before_action :authenticate_user!
+  before_action :require_not_suspended!
+
+  skip_before_action :require_functional!
 
   def show
     @confirmation = Form::DeleteConfirmation.new
@@ -29,4 +32,8 @@ class Settings::DeletesController < Settings::BaseController
   def delete_params
     params.require(:form_delete_confirmation).permit(:password)
   end
+
+  def require_not_suspended!
+    forbidden if current_account.suspended?
+  end
 end
diff --git a/app/controllers/settings/sessions_controller.rb b/app/controllers/settings/sessions_controller.rb
index 84ebb21f2..df5ace803 100644
--- a/app/controllers/settings/sessions_controller.rb
+++ b/app/controllers/settings/sessions_controller.rb
@@ -4,6 +4,8 @@ class Settings::SessionsController < Settings::BaseController
   before_action :authenticate_user!
   before_action :set_session, only: :destroy
 
+  skip_before_action :require_functional!
+
   def destroy
     @session.destroy!
     flash[:notice] = I18n.t('sessions.revoke_success')
diff --git a/app/controllers/settings/two_factor_authentication/confirmations_controller.rb b/app/controllers/settings/two_factor_authentication/confirmations_controller.rb
index 02652a36c..3145e092d 100644
--- a/app/controllers/settings/two_factor_authentication/confirmations_controller.rb
+++ b/app/controllers/settings/two_factor_authentication/confirmations_controller.rb
@@ -8,6 +8,8 @@ module Settings
       before_action :authenticate_user!
       before_action :ensure_otp_secret
 
+      skip_before_action :require_functional!
+
       def new
         prepare_two_factor_form
       end
diff --git a/app/controllers/settings/two_factor_authentication/recovery_codes_controller.rb b/app/controllers/settings/two_factor_authentication/recovery_codes_controller.rb
index 874bf532b..09a759860 100644
--- a/app/controllers/settings/two_factor_authentication/recovery_codes_controller.rb
+++ b/app/controllers/settings/two_factor_authentication/recovery_codes_controller.rb
@@ -7,6 +7,8 @@ module Settings
 
       before_action :authenticate_user!
 
+      skip_before_action :require_functional!
+
       def create
         @recovery_codes = current_user.generate_otp_backup_codes!
         current_user.save!
diff --git a/app/controllers/settings/two_factor_authentications_controller.rb b/app/controllers/settings/two_factor_authentications_controller.rb
index e12c43074..6904076e4 100644
--- a/app/controllers/settings/two_factor_authentications_controller.rb
+++ b/app/controllers/settings/two_factor_authentications_controller.rb
@@ -7,6 +7,8 @@ module Settings
     before_action :authenticate_user!
     before_action :verify_otp_required, only: [:create]
 
+    skip_before_action :require_functional!
+
     def show
       @confirmation = Form::TwoFactorConfirmation.new
     end
diff --git a/app/javascript/styles/mastodon/admin.scss b/app/javascript/styles/mastodon/admin.scss
index 373a10260..f625bc139 100644
--- a/app/javascript/styles/mastodon/admin.scss
+++ b/app/javascript/styles/mastodon/admin.scss
@@ -204,29 +204,6 @@ $content-width: 840px;
         border: 0;
       }
     }
-
-    .muted-hint {
-      color: $darker-text-color;
-
-      a {
-        color: $highlight-text-color;
-      }
-    }
-
-    .positive-hint {
-      color: $valid-value-color;
-      font-weight: 500;
-    }
-
-    .negative-hint {
-      color: $error-value-color;
-      font-weight: 500;
-    }
-
-    .neutral-hint {
-      color: $dark-text-color;
-      font-weight: 500;
-    }
   }
 
   @media screen and (max-width: $no-columns-breakpoint) {
@@ -249,6 +226,41 @@ $content-width: 840px;
   }
 }
 
+hr.spacer {
+  width: 100%;
+  border: 0;
+  margin: 20px 0;
+  height: 1px;
+}
+
+.muted-hint {
+  color: $darker-text-color;
+
+  a {
+    color: $highlight-text-color;
+  }
+}
+
+.positive-hint {
+  color: $valid-value-color;
+  font-weight: 500;
+}
+
+.negative-hint {
+  color: $error-value-color;
+  font-weight: 500;
+}
+
+.neutral-hint {
+  color: $dark-text-color;
+  font-weight: 500;
+}
+
+.warning-hint {
+  color: $gold-star;
+  font-weight: 500;
+}
+
 .filters {
   display: flex;
   flex-wrap: wrap;
diff --git a/app/javascript/styles/mastodon/forms.scss b/app/javascript/styles/mastodon/forms.scss
index 456ee4e0d..ac99124ea 100644
--- a/app/javascript/styles/mastodon/forms.scss
+++ b/app/javascript/styles/mastodon/forms.scss
@@ -300,6 +300,13 @@ code {
     }
   }
 
+  .input.static .label_input__wrapper {
+    font-size: 16px;
+    padding: 10px;
+    border: 1px solid $dark-text-color;
+    border-radius: 4px;
+  }
+
   input[type=text],
   input[type=number],
   input[type=email],
diff --git a/app/models/concerns/omniauthable.rb b/app/models/concerns/omniauthable.rb
index 283033083..b9c124841 100644
--- a/app/models/concerns/omniauthable.rb
+++ b/app/models/concerns/omniauthable.rb
@@ -43,7 +43,7 @@ module Omniauthable
       # Check if the user exists with provided email if the provider gives us a
       # verified email.  If no verified email was provided or the user already
       # exists, we assign a temporary email and ask the user to verify it on
-      # the next step via Auth::ConfirmationsController.finish_signup
+      # the next step via Auth::SetupController.show
 
       user = User.new(user_params_from_auth(auth))
       user.account.avatar_remote_url = auth.info.image if auth.info.image =~ /\A#{URI.regexp(%w(http https))}\z/
diff --git a/app/models/user.rb b/app/models/user.rb
index 31c99630c..474c77293 100644
--- a/app/models/user.rb
+++ b/app/models/user.rb
@@ -161,7 +161,11 @@ class User < ApplicationRecord
   end
 
   def active_for_authentication?
-    super && approved?
+    true
+  end
+
+  def functional?
+    confirmed? && approved? && !disabled? && !account.suspended?
   end
 
   def inactive_message
diff --git a/app/views/auth/confirmations/finish_signup.html.haml b/app/views/auth/confirmations/finish_signup.html.haml
deleted file mode 100644
index 9d09b74e1..000000000
--- a/app/views/auth/confirmations/finish_signup.html.haml
+++ /dev/null
@@ -1,15 +0,0 @@
-- content_for :page_title do
-  = t('auth.confirm_email')
-
-= simple_form_for(current_user, as: 'user', url: finish_signup_path, html: { role: 'form'}) do |f|
-  - if @show_errors && current_user.errors.any?
-    #error_explanation
-      - current_user.errors.full_messages.each do |msg|
-        = msg
-        %br
-
-  .fields-group
-    = f.input :email, wrapper: :with_label, required: true, hint: false
-
-  .actions
-    = f.submit t('auth.confirm_email'), class: 'button'
diff --git a/app/views/auth/registrations/_sessions.html.haml b/app/views/auth/registrations/_sessions.html.haml
index d7d96a1bb..395e36a9f 100644
--- a/app/views/auth/registrations/_sessions.html.haml
+++ b/app/views/auth/registrations/_sessions.html.haml
@@ -1,6 +1,8 @@
-%h4= t 'sessions.title'
+%h3= t 'sessions.title'
 %p.muted-hint= t 'sessions.explanation'
 
+%hr.spacer/
+
 .table-wrapper
   %table.table.inline-table
     %thead
diff --git a/app/views/auth/registrations/_status.html.haml b/app/views/auth/registrations/_status.html.haml
new file mode 100644
index 000000000..b38a83d67
--- /dev/null
+++ b/app/views/auth/registrations/_status.html.haml
@@ -0,0 +1,16 @@
+%h3= t('auth.status.account_status')
+
+- if @user.account.suspended?
+  %span.negative-hint= t('user_mailer.warning.explanation.suspend')
+- elsif @user.disabled?
+  %span.negative-hint= t('user_mailer.warning.explanation.disable')
+- elsif @user.account.silenced?
+  %span.warning-hint= t('user_mailer.warning.explanation.silence')
+- elsif !@user.confirmed?
+  %span.warning-hint= t('auth.status.confirming')
+- elsif !@user.approved?
+  %span.warning-hint= t('auth.status.pending')
+- else
+  %span.positive-hint= t('auth.status.functional')
+
+%hr.spacer/
diff --git a/app/views/auth/registrations/edit.html.haml b/app/views/auth/registrations/edit.html.haml
index 694461fdf..710ee5c68 100644
--- a/app/views/auth/registrations/edit.html.haml
+++ b/app/views/auth/registrations/edit.html.haml
@@ -1,25 +1,28 @@
 - content_for :page_title do
-  = t('auth.security')
+  = t('settings.account_settings')
+
+= render 'status'
+
+%h3= t('auth.security')
 
 = simple_form_for(resource, as: resource_name, url: registration_path(resource_name), html: { method: :put, class: 'auth_edit' }) do |f|
   = render 'shared/error_messages', object: resource
 
   - if !use_seamless_external_login? || resource.encrypted_password.present?
-    .fields-group
-      = f.input :email, wrapper: :with_label, input_html: { 'aria-label' => t('simple_form.labels.defaults.email') }, required: true, hint: false
-
-    .fields-group
-      = f.input :current_password, wrapper: :with_label, input_html: { 'aria-label' => t('simple_form.labels.defaults.current_password'), :autocomplete => 'off' }, required: true
-
-    .fields-group
-      = f.input :password, wrapper: :with_label, label: t('simple_form.labels.defaults.new_password'), input_html: { 'aria-label' => t('simple_form.labels.defaults.new_password'), :autocomplete => 'off' }, hint: false
-
-    .fields-group
-      = f.input :password_confirmation, wrapper: :with_label, label: t('simple_form.labels.defaults.confirm_new_password'), input_html: { 'aria-label' => t('simple_form.labels.defaults.confirm_new_password'), :autocomplete => 'off' }
-
+    .fields-row
+      .fields-row__column.fields-group.fields-row__column-6
+        = f.input :email, wrapper: :with_label, input_html: { 'aria-label' => t('simple_form.labels.defaults.email') }, required: true, disabled: current_account.suspended?
+      .fields-row__column.fields-group.fields-row__column-6
+        = f.input :current_password, wrapper: :with_label, input_html: { 'aria-label' => t('simple_form.labels.defaults.current_password'), :autocomplete => 'off' }, required: true, disabled: current_account.suspended?
+
+    .fields-row
+      .fields-row__column.fields-group.fields-row__column-6
+        = f.input :password, wrapper: :with_label, label: t('simple_form.labels.defaults.new_password'), input_html: { 'aria-label' => t('simple_form.labels.defaults.new_password'), :autocomplete => 'off' }, hint: t('simple_form.hints.defaults.password'), disabled: current_account.suspended?
+      .fields-row__column.fields-group.fields-row__column-6
+        = f.input :password_confirmation, wrapper: :with_label, label: t('simple_form.labels.defaults.confirm_new_password'), input_html: { 'aria-label' => t('simple_form.labels.defaults.confirm_new_password'), :autocomplete => 'off' }, disabled: current_account.suspended?
 
     .actions
-      = f.button :button, t('generic.save_changes'), type: :submit
+      = f.button :button, t('generic.save_changes'), type: :submit, class: 'button', disabled: current_account.suspended?
   - else
     %p.hint= t('users.seamless_external_login')
 
@@ -27,7 +30,7 @@
 
 = render 'sessions'
 
-- if open_deletion?
+- if open_deletion? && !current_account.suspended?
   %hr.spacer/
-  %h4= t('auth.delete_account')
+  %h3= t('auth.delete_account')
   %p.muted-hint= t('auth.delete_account_html', path: settings_delete_path)
diff --git a/app/views/auth/setup/show.html.haml b/app/views/auth/setup/show.html.haml
new file mode 100644
index 000000000..8bb44ca7f
--- /dev/null
+++ b/app/views/auth/setup/show.html.haml
@@ -0,0 +1,23 @@
+- content_for :page_title do
+  = t('auth.setup.title')
+
+- if missing_email?
+  = simple_form_for(@user, url: auth_setup_path) do |f|
+    = render 'shared/error_messages', object: @user
+
+    .fields-group
+      %p.hint= t('auth.setup.email_below_hint_html')
+
+    .fields-group
+      = f.input :email, required: true, hint: false, input_html: { 'aria-label' => t('simple_form.labels.defaults.email'), :autocomplete => 'off' }
+
+    .actions
+      = f.submit t('admin.accounts.change_email.label'), class: 'button'
+- else
+  .simple_form
+    %p.hint= t('auth.setup.email_settings_hint_html', email: content_tag(:strong, @user.email))
+
+.form-footer
+  %ul.no-list
+    %li= link_to t('settings.account_settings'), edit_user_registration_path
+    %li= link_to t('auth.logout'), destroy_user_session_path, data: { method: :delete }
diff --git a/app/views/oauth/authorized_applications/index.html.haml b/app/views/oauth/authorized_applications/index.html.haml
index 19af5f55d..7203d758d 100644
--- a/app/views/oauth/authorized_applications/index.html.haml
+++ b/app/views/oauth/authorized_applications/index.html.haml
@@ -17,7 +17,7 @@
               = application.name
             - else
               = link_to application.name, application.website, target: '_blank', rel: 'noopener'
-          %th!= application.scopes.map { |scope| t(scope, scope: [:doorkeeper, :scopes]) }.join('<br />')
+          %th!= application.scopes.map { |scope| t(scope, scope: [:doorkeeper, :scopes]) }.join(', ')
           %td= l application.created_at
           %td
             - unless application.superapp?
diff --git a/config/locales/en.yml b/config/locales/en.yml
index 89c52b84a..9e1be87be 100644
--- a/config/locales/en.yml
+++ b/config/locales/en.yml
@@ -524,7 +524,6 @@ en:
     apply_for_account: Request an invite
     change_password: Password
     checkbox_agreement_html: I agree to the <a href="%{rules_path}" target="_blank">server rules</a> and <a href="%{terms_path}" target="_blank">terms of service</a>
-    confirm_email: Confirm email
     delete_account: Delete account
     delete_account_html: If you wish to delete your account, you can <a href="%{path}">proceed here</a>. You will be asked for confirmation.
     didnt_get_confirmation: Didn't receive confirmation instructions?
@@ -544,6 +543,14 @@ en:
     reset_password: Reset password
     security: Security
     set_new_password: Set new password
+    setup:
+      email_below_hint_html: If the below e-mail address is incorrect, you can change it here and receive a new confirmation e-mail.
+      email_settings_hint_html: The confirmation e-mail was sent to %{email}. If that e-mail address is not correct, you can change it in account settings.
+      title: Setup
+    status:
+      account_status: Account status
+      confirming: Waiting for e-mail confirmation to be completed.
+      pending: Your application is pending review by our staff. This may take some time. You will receive an e-mail if your application is approved.
     trouble_logging_in: Trouble logging in?
   authorize_follow:
     already_following: You are already following this account
diff --git a/config/routes.rb b/config/routes.rb
index 27b536641..b6c215888 100644
--- a/config/routes.rb
+++ b/config/routes.rb
@@ -34,7 +34,10 @@ Rails.application.routes.draw do
 
   devise_scope :user do
     get '/invite/:invite_code', to: 'auth/registrations#new', as: :public_invite
-    match '/auth/finish_signup' => 'auth/confirmations#finish_signup', via: [:get, :patch], as: :finish_signup
+
+    namespace :auth do
+      resource :setup, only: [:show, :update], controller: :setup
+    end
   end
 
   devise_for :users, path: 'auth', controllers: {
diff --git a/db/seeds.rb b/db/seeds.rb
index b112cf073..0bfb5d0db 100644
--- a/db/seeds.rb
+++ b/db/seeds.rb
@@ -1,4 +1,4 @@
-Doorkeeper::Application.create!(name: 'Web', superapp: true, redirect_uri: Doorkeeper.configuration.native_redirect_uri, scopes: 'read write follow')
+Doorkeeper::Application.create!(name: 'Web', superapp: true, redirect_uri: Doorkeeper.configuration.native_redirect_uri, scopes: 'read write follow push')
 
 domain = ENV['LOCAL_DOMAIN'] || Rails.configuration.x.local_domain
 account = Account.find_or_initialize_by(id: -99, actor_type: 'Application', locked: true, username: domain)
diff --git a/spec/controllers/api/base_controller_spec.rb b/spec/controllers/api/base_controller_spec.rb
index 750ccc8cf..05a42d1c1 100644
--- a/spec/controllers/api/base_controller_spec.rb
+++ b/spec/controllers/api/base_controller_spec.rb
@@ -15,7 +15,7 @@ describe Api::BaseController do
     end
   end
 
-  describe 'Forgery protection' do
+  describe 'forgery protection' do
     before do
       routes.draw { post 'success' => 'api/base#success' }
     end
@@ -27,7 +27,45 @@ describe Api::BaseController do
     end
   end
 
-  describe 'Error handling' do
+  describe 'non-functional accounts handling' do
+    let(:user)  { Fabricate(:user, account: Fabricate(:account, username: 'alice')) }
+    let(:token) { Fabricate(:accessible_access_token, resource_owner_id: user.id, scopes: 'read') }
+
+    controller do
+      before_action :require_user!
+    end
+
+    before do
+      routes.draw { post 'success' => 'api/base#success' }
+      allow(controller).to receive(:doorkeeper_token) { token }
+    end
+
+    it 'returns http forbidden for unconfirmed accounts' do
+      user.update(confirmed_at: nil)
+      post 'success'
+      expect(response).to have_http_status(403)
+    end
+
+    it 'returns http forbidden for pending accounts' do
+      user.update(approved: false)
+      post 'success'
+      expect(response).to have_http_status(403)
+    end
+
+    it 'returns http forbidden for disabled accounts' do
+      user.update(disabled: true)
+      post 'success'
+      expect(response).to have_http_status(403)
+    end
+
+    it 'returns http forbidden for suspended accounts' do
+      user.account.suspend!
+      post 'success'
+      expect(response).to have_http_status(403)
+    end
+  end
+
+  describe 'error handling' do
     ERRORS_WITH_CODES = {
       ActiveRecord::RecordInvalid => 422,
       Mastodon::ValidationError => 422,
diff --git a/spec/controllers/application_controller_spec.rb b/spec/controllers/application_controller_spec.rb
index 27946b60f..1811500df 100644
--- a/spec/controllers/application_controller_spec.rb
+++ b/spec/controllers/application_controller_spec.rb
@@ -187,10 +187,10 @@ describe ApplicationController, type: :controller do
       expect(response).to have_http_status(200)
     end
 
-    it 'returns http 403 if user who signed in is suspended' do
+    it 'redirects to account status page' do
       sign_in(Fabricate(:user, account: Fabricate(:account, suspended: true)))
       get 'success'
-      expect(response).to have_http_status(403)
+      expect(response).to redirect_to(edit_user_registration_path)
     end
   end
 
diff --git a/spec/controllers/auth/confirmations_controller_spec.rb b/spec/controllers/auth/confirmations_controller_spec.rb
index e9a471fc5..0b6b74ff9 100644
--- a/spec/controllers/auth/confirmations_controller_spec.rb
+++ b/spec/controllers/auth/confirmations_controller_spec.rb
@@ -50,45 +50,4 @@ describe Auth::ConfirmationsController, type: :controller do
       end
     end
   end
-
-  describe 'GET #finish_signup' do
-    subject { get :finish_signup }
-
-    let(:user) { Fabricate(:user) }
-    before do
-      sign_in user, scope: :user
-      @request.env['devise.mapping'] = Devise.mappings[:user]
-    end
-
-    it 'renders finish_signup' do
-      is_expected.to render_template :finish_signup
-      expect(assigns(:user)).to have_attributes id: user.id
-    end
-  end
-
-  describe 'PATCH #finish_signup' do
-    subject { patch :finish_signup, params: { user: { email: email } } }
-
-    let(:user) { Fabricate(:user) }
-    before do
-      sign_in user, scope: :user
-      @request.env['devise.mapping'] = Devise.mappings[:user]
-    end
-
-    context 'when email is valid' do
-      let(:email) { 'new_' + user.email }
-
-      it 'redirects to root_path' do
-        is_expected.to redirect_to root_path
-      end
-    end
-
-    context 'when email is invalid' do
-      let(:email) { '' }
-
-      it 'renders finish_signup' do
-        is_expected.to render_template :finish_signup
-      end
-    end
-  end
 end
diff --git a/spec/controllers/auth/registrations_controller_spec.rb b/spec/controllers/auth/registrations_controller_spec.rb
index a4337039e..3e11b34b5 100644
--- a/spec/controllers/auth/registrations_controller_spec.rb
+++ b/spec/controllers/auth/registrations_controller_spec.rb
@@ -46,6 +46,15 @@ RSpec.describe Auth::RegistrationsController, type: :controller do
       post :update
       expect(response).to have_http_status(200)
     end
+
+    context 'when suspended' do
+      it 'returns http forbidden' do
+        request.env["devise.mapping"] = Devise.mappings[:user]
+        sign_in(Fabricate(:user, account_attributes: { username: 'test', suspended_at: Time.now.utc }), scope: :user)
+        post :update
+        expect(response).to have_http_status(403)
+      end
+    end
   end
 
   describe 'GET #new' do
@@ -94,9 +103,9 @@ RSpec.describe Auth::RegistrationsController, type: :controller do
         post :create, params: { user: { account_attributes: { username: 'test' }, email: 'test@example.com', password: '12345678', password_confirmation: '12345678' } }
       end
 
-      it 'redirects to login page' do
+      it 'redirects to setup' do
         subject
-        expect(response).to redirect_to new_user_session_path
+        expect(response).to redirect_to auth_setup_path
       end
 
       it 'creates user' do
@@ -120,9 +129,9 @@ RSpec.describe Auth::RegistrationsController, type: :controller do
         post :create, params: { user: { account_attributes: { username: 'test' }, email: 'test@example.com', password: '12345678', password_confirmation: '12345678' } }
       end
 
-      it 'redirects to login page' do
+      it 'redirects to setup' do
         subject
-        expect(response).to redirect_to new_user_session_path
+        expect(response).to redirect_to auth_setup_path
       end
 
       it 'creates user' do
@@ -148,9 +157,9 @@ RSpec.describe Auth::RegistrationsController, type: :controller do
         post :create, params: { user: { account_attributes: { username: 'test' }, email: 'test@example.com', password: '12345678', password_confirmation: '12345678', 'invite_code': invite.code } }
       end
 
-      it 'redirects to login page' do
+      it 'redirects to setup' do
         subject
-        expect(response).to redirect_to new_user_session_path
+        expect(response).to redirect_to auth_setup_path
       end
 
       it 'creates user' do
@@ -176,9 +185,9 @@ RSpec.describe Auth::RegistrationsController, type: :controller do
         post :create, params: { user: { account_attributes: { username: 'test' }, email: 'test@example.com', password: '12345678', password_confirmation: '12345678', 'invite_code': invite.code } }
       end
 
-      it 'redirects to login page' do
+      it 'redirects to setup' do
         subject
-        expect(response).to redirect_to new_user_session_path
+        expect(response).to redirect_to auth_setup_path
       end
 
       it 'creates user' do
diff --git a/spec/controllers/auth/sessions_controller_spec.rb b/spec/controllers/auth/sessions_controller_spec.rb
index 71fcc1a6e..87ef4f2bb 100644
--- a/spec/controllers/auth/sessions_controller_spec.rb
+++ b/spec/controllers/auth/sessions_controller_spec.rb
@@ -160,8 +160,8 @@ RSpec.describe Auth::SessionsController, type: :controller do
         let(:unconfirmed_user) { user.tap { |u| u.update!(confirmed_at: nil) } }
         let(:accept_language) { 'fr' }
 
-        it 'shows a translated login error' do
-          expect(flash[:alert]).to eq(I18n.t('devise.failure.unconfirmed', locale: accept_language))
+        it 'redirects to home' do
+          expect(response).to redirect_to(root_path)
         end
       end
 
diff --git a/spec/controllers/settings/deletes_controller_spec.rb b/spec/controllers/settings/deletes_controller_spec.rb
index 35fd64e9b..996872efd 100644
--- a/spec/controllers/settings/deletes_controller_spec.rb
+++ b/spec/controllers/settings/deletes_controller_spec.rb
@@ -15,6 +15,15 @@ describe Settings::DeletesController do
         get :show
         expect(response).to have_http_status(200)
       end
+
+      context 'when suspended' do
+        let(:user) { Fabricate(:user, account_attributes: { username: 'alice', suspended_at: Time.now.utc }) }
+
+        it 'returns http forbidden' do
+          get :show
+          expect(response).to have_http_status(403)
+        end
+      end
     end
 
     context 'when not signed in' do
@@ -49,6 +58,14 @@ describe Settings::DeletesController do
         it 'marks account as suspended' do
           expect(user.account.reload).to be_suspended
         end
+
+        context 'when suspended' do
+          let(:user) { Fabricate(:user, account_attributes: { username: 'alice', suspended_at: Time.now.utc }) }
+
+          it 'returns http forbidden' do
+            expect(response).to have_http_status(403)
+          end
+        end
       end
 
       context 'with incorrect password' do
diff --git a/spec/features/log_in_spec.rb b/spec/features/log_in_spec.rb
index 53a1f9b12..f6c26cd0f 100644
--- a/spec/features/log_in_spec.rb
+++ b/spec/features/log_in_spec.rb
@@ -31,12 +31,12 @@ feature "Log in" do
   context do
     given(:confirmed_at) { nil }
 
-    scenario "A unconfirmed user is not able to log in" do
+    scenario "A unconfirmed user is able to log in" do
       fill_in "user_email", with: email
       fill_in "user_password", with: password
       click_on I18n.t('auth.login')
 
-      is_expected.to have_css(".flash-message", text: failure_message("unconfirmed"))
+      is_expected.to have_css("div.admin-wrapper")
     end
   end
 
diff --git a/spec/models/user_spec.rb b/spec/models/user_spec.rb
index 856254ce4..d7c0b5359 100644
--- a/spec/models/user_spec.rb
+++ b/spec/models/user_spec.rb
@@ -506,7 +506,7 @@ RSpec.describe User, type: :model do
       context 'when user is not confirmed' do
         let(:confirmed_at) { nil }
 
-        it { is_expected.to be false }
+        it { is_expected.to be true }
       end
     end
 
@@ -522,7 +522,7 @@ RSpec.describe User, type: :model do
       context 'when user is not confirmed' do
         let(:confirmed_at) { nil }
 
-        it { is_expected.to be false }
+        it { is_expected.to be true }
       end
     end
   end
-- 
cgit 


From 94f5c714f11248df6a2b793f47cdb30028f67fca Mon Sep 17 00:00:00 2001
From: Rey Tucker <git@reytucker.us>
Date: Wed, 24 Jul 2019 08:19:17 -0400
Subject:  Don't delete periods when validating username uniqueness (#11392)
 (#11400)

* Check to make sure usernames with '.' cannot be created

* Add test for instance actor account name conflicts

This makes sure that migration 20190715164535_add_instance_actor
won't fail if there's already an account that is named the same
as the domain (minus the .)

* Put the test into the correct context...

* Add another test to split this into two validations

* Don't delete periods when validating username uniqueness (#11392)

The 20190715164535_add_instance_actor migration fails if there's
already a username similar to the domain name, e.g. if you are
'vulpine.club' and have a user named 'vulpineclub', validation
fails.

Upon further review, usernames with periods are dropped by the
regular expression in the Account class, so we don't need to
worry about it here.

Fixes #11392
---
 app/validators/unique_username_validator.rb |  4 +++-
 spec/models/account_spec.rb                 | 17 +++++++++++++++++
 2 files changed, 20 insertions(+), 1 deletion(-)

(limited to 'spec')

diff --git a/app/validators/unique_username_validator.rb b/app/validators/unique_username_validator.rb
index fb67105dd..4e24e3f5f 100644
--- a/app/validators/unique_username_validator.rb
+++ b/app/validators/unique_username_validator.rb
@@ -1,10 +1,12 @@
 # frozen_string_literal: true
 
+# See also: USERNAME_RE in the Account class
+
 class UniqueUsernameValidator < ActiveModel::Validator
   def validate(account)
     return if account.username.nil?
 
-    normalized_username = account.username.downcase.delete('.')
+    normalized_username = account.username.downcase
 
     scope = Account.where(domain: nil).where('lower(username) = ?', normalized_username)
     scope = scope.where.not(id: account.id) if account.persisted?
diff --git a/spec/models/account_spec.rb b/spec/models/account_spec.rb
index 6495a6193..3a17d540a 100644
--- a/spec/models/account_spec.rb
+++ b/spec/models/account_spec.rb
@@ -583,12 +583,29 @@ RSpec.describe Account, type: :model do
         expect(account.valid?).to be true
       end
 
+      it 'is valid if we are creating an instance actor account with a period' do
+        account = Fabricate.build(:account, id: -99, actor_type: 'Application', locked: true, username: 'example.com')
+        expect(account.valid?).to be true
+      end
+
+      it 'is valid if we are creating a possibly-conflicting instance actor account' do
+        account_1 = Fabricate(:account, username: 'examplecom')
+        account_2 = Fabricate.build(:account, id: -99, actor_type: 'Application', locked: true, username: 'example.com')
+        expect(account_2.valid?).to be true
+      end
+
       it 'is invalid if the username doesn\'t only contains letters, numbers and underscores' do
         account = Fabricate.build(:account, username: 'the-doctor')
         account.valid?
         expect(account).to model_have_error_on_field(:username)
       end
 
+      it 'is invalid if the username contains a period' do
+        account = Fabricate.build(:account, username: 'the.doctor')
+        account.valid?
+        expect(account).to model_have_error_on_field(:username)
+      end
+
       it 'is invalid if the username is longer then 30 characters' do
         account = Fabricate.build(:account, username: Faker::Lorem.characters(31))
         account.valid?
-- 
cgit 


From 92569ffde81a3fe768fd8773fdab5d1a927f3f88 Mon Sep 17 00:00:00 2001
From: ThibG <thib@sitedethib.com>
Date: Fri, 26 Jul 2019 18:55:33 +0200
Subject: Fix invites not being disabled upon account suspension (#11412)

* Disable invite links from disabled/suspended users

* Add has_many invites relationship to users

* Destroy unused invites when suspending an account
---
 app/controllers/invites_controller.rb   |  2 +-
 app/models/invite.rb                    |  4 ++--
 app/models/user.rb                      |  1 +
 app/services/suspend_account_service.rb |  1 +
 spec/models/invite_spec.rb              | 16 +++++++++++-----
 5 files changed, 16 insertions(+), 8 deletions(-)

(limited to 'spec')

diff --git a/app/controllers/invites_controller.rb b/app/controllers/invites_controller.rb
index fdb3a0962..de5280305 100644
--- a/app/controllers/invites_controller.rb
+++ b/app/controllers/invites_controller.rb
@@ -39,7 +39,7 @@ class InvitesController < ApplicationController
   private
 
   def invites
-    Invite.where(user: current_user).order(id: :desc)
+    current_user.invites.order(id: :desc)
   end
 
   def resource_params
diff --git a/app/models/invite.rb b/app/models/invite.rb
index fe2322462..02ab8e0b2 100644
--- a/app/models/invite.rb
+++ b/app/models/invite.rb
@@ -17,7 +17,7 @@
 class Invite < ApplicationRecord
   include Expireable
 
-  belongs_to :user
+  belongs_to :user, inverse_of: :invites
   has_many :users, inverse_of: :invite
 
   scope :available, -> { where(expires_at: nil).or(where('expires_at >= ?', Time.now.utc)) }
@@ -25,7 +25,7 @@ class Invite < ApplicationRecord
   before_validation :set_code
 
   def valid_for_use?
-    (max_uses.nil? || uses < max_uses) && !expired?
+    (max_uses.nil? || uses < max_uses) && !expired? && !(user.nil? || user.disabled?)
   end
 
   private
diff --git a/app/models/user.rb b/app/models/user.rb
index 474c77293..6806c0362 100644
--- a/app/models/user.rb
+++ b/app/models/user.rb
@@ -73,6 +73,7 @@ class User < ApplicationRecord
 
   has_many :applications, class_name: 'Doorkeeper::Application', as: :owner
   has_many :backups, inverse_of: :user
+  has_many :invites, inverse_of: :user
 
   has_one :invite_request, class_name: 'UserInviteRequest', inverse_of: :user, dependent: :destroy
   accepts_nested_attributes_for :invite_request, reject_if: ->(attributes) { attributes['text'].blank? }
diff --git a/app/services/suspend_account_service.rb b/app/services/suspend_account_service.rb
index 00cffcdfc..902af376c 100644
--- a/app/services/suspend_account_service.rb
+++ b/app/services/suspend_account_service.rb
@@ -64,6 +64,7 @@ class SuspendAccountService < BaseService
       @account.user.destroy
     else
       @account.user.disable!
+      @account.user.invites.where(uses: 0).destroy_all
     end
   end
 
diff --git a/spec/models/invite_spec.rb b/spec/models/invite_spec.rb
index 0ba1dccb3..30abfb86b 100644
--- a/spec/models/invite_spec.rb
+++ b/spec/models/invite_spec.rb
@@ -3,27 +3,33 @@ require 'rails_helper'
 RSpec.describe Invite, type: :model do
   describe '#valid_for_use?' do
     it 'returns true when there are no limitations' do
-      invite = Invite.new(max_uses: nil, expires_at: nil)
+      invite = Fabricate(:invite, max_uses: nil, expires_at: nil)
       expect(invite.valid_for_use?).to be true
     end
 
     it 'returns true when not expired' do
-      invite = Invite.new(max_uses: nil, expires_at: 1.hour.from_now)
+      invite = Fabricate(:invite, max_uses: nil, expires_at: 1.hour.from_now)
       expect(invite.valid_for_use?).to be true
     end
 
     it 'returns false when expired' do
-      invite = Invite.new(max_uses: nil, expires_at: 1.hour.ago)
+      invite = Fabricate(:invite, max_uses: nil, expires_at: 1.hour.ago)
       expect(invite.valid_for_use?).to be false
     end
 
     it 'returns true when uses still available' do
-      invite = Invite.new(max_uses: 250, uses: 249, expires_at: nil)
+      invite = Fabricate(:invite, max_uses: 250, uses: 249, expires_at: nil)
       expect(invite.valid_for_use?).to be true
     end
 
     it 'returns false when maximum uses reached' do
-      invite = Invite.new(max_uses: 250, uses: 250, expires_at: nil)
+      invite = Fabricate(:invite, max_uses: 250, uses: 250, expires_at: nil)
+      expect(invite.valid_for_use?).to be false
+    end
+
+    it 'returns false when invite creator has been disabled' do
+      invite = Fabricate(:invite, max_uses: nil, expires_at: nil)
+      SuspendAccountService.new.call(invite.user.account)
       expect(invite.valid_for_use?).to be false
     end
   end
-- 
cgit 


From b9fbcbfe4e0a15fcf8a457ce17ea080f0eb939fc Mon Sep 17 00:00:00 2001
From: Eugen Rochko <eugen@zeonfederated.com>
Date: Sat, 27 Jul 2019 04:42:08 +0200
Subject: Add search syntax for operators and phrases (#11411)

---
 Gemfile                              |  1 +
 Gemfile.lock                         |  2 +
 app/lib/search_query_parser.rb       | 14 ++++++
 app/lib/search_query_transformer.rb  | 86 ++++++++++++++++++++++++++++++++++++
 app/services/search_service.rb       |  9 ++--
 spec/services/search_service_spec.rb |  6 +--
 6 files changed, 112 insertions(+), 6 deletions(-)
 create mode 100644 app/lib/search_query_parser.rb
 create mode 100644 app/lib/search_query_transformer.rb

(limited to 'spec')

diff --git a/Gemfile b/Gemfile
index e969c4ae0..96eb44af7 100644
--- a/Gemfile
+++ b/Gemfile
@@ -64,6 +64,7 @@ gem 'nsa', '~> 0.2'
 gem 'oj', '~> 3.8'
 gem 'ostatus2', '~> 2.0'
 gem 'ox', '~> 2.11'
+gem 'parslet'
 gem 'posix-spawn', git: 'https://github.com/rtomayko/posix-spawn', ref: '58465d2e213991f8afb13b984854a49fcdcc980c'
 gem 'pundit', '~> 2.0'
 gem 'premailer-rails'
diff --git a/Gemfile.lock b/Gemfile.lock
index f435b3a35..5adae6f9c 100644
--- a/Gemfile.lock
+++ b/Gemfile.lock
@@ -404,6 +404,7 @@ GEM
       parallel
     parser (2.6.3.0)
       ast (~> 2.4.0)
+    parslet (1.8.2)
     pastel (0.7.2)
       equatable (~> 0.5.0)
       tty-color (~> 0.4.0)
@@ -724,6 +725,7 @@ DEPENDENCIES
   paperclip (~> 6.0)
   paperclip-av-transcoder (~> 0.6)
   parallel_tests (~> 2.29)
+  parslet
   pg (~> 1.1)
   pghero (~> 2.2)
   pkg-config (~> 1.3)
diff --git a/app/lib/search_query_parser.rb b/app/lib/search_query_parser.rb
new file mode 100644
index 000000000..405ad15b8
--- /dev/null
+++ b/app/lib/search_query_parser.rb
@@ -0,0 +1,14 @@
+# frozen_string_literal: true
+
+class SearchQueryParser < Parslet::Parser
+  rule(:term)     { match('[^\s":]').repeat(1).as(:term) }
+  rule(:quote)    { str('"') }
+  rule(:colon)    { str(':') }
+  rule(:space)    { match('\s').repeat(1) }
+  rule(:operator) { (str('+') | str('-')).as(:operator) }
+  rule(:prefix)   { (term >> colon).as(:prefix) }
+  rule(:phrase)   { (quote >> (term >> space.maybe).repeat >> quote).as(:phrase) }
+  rule(:clause)   { (prefix.maybe >> operator.maybe >> (phrase | term)).as(:clause) }
+  rule(:query)    { (clause >> space.maybe).repeat.as(:query) }
+  root(:query)
+end
diff --git a/app/lib/search_query_transformer.rb b/app/lib/search_query_transformer.rb
new file mode 100644
index 000000000..2c4144790
--- /dev/null
+++ b/app/lib/search_query_transformer.rb
@@ -0,0 +1,86 @@
+# frozen_string_literal: true
+
+class SearchQueryTransformer < Parslet::Transform
+  class Query
+    attr_reader :should_clauses, :must_not_clauses, :must_clauses
+
+    def initialize(clauses)
+      grouped = clauses.chunk(&:operator).to_h
+      @should_clauses = grouped.fetch(:should, [])
+      @must_not_clauses = grouped.fetch(:must_not, [])
+      @must_clauses = grouped.fetch(:must, [])
+    end
+
+    def apply(search)
+      should_clauses.each { |clause| search = search.query.should(clause_to_query(clause)) }
+      must_clauses.each { |clause| search = search.query.must(clause_to_query(clause)) }
+      must_not_clauses.each { |clause| search = search.query.must_not(clause_to_query(clause)) }
+      search.query.minimum_should_match(1)
+    end
+
+    private
+
+    def clause_to_query(clause)
+      case clause
+      when TermClause
+        { multi_match: { type: 'most_fields', query: clause.term, fields: ['text', 'text.stemmed'] } }
+      when PhraseClause
+        { match_phrase: { text: { query: clause.phrase } } }
+      else
+        raise "Unexpected clause type: #{clause}"
+      end
+    end
+  end
+
+  class Operator
+    class << self
+      def symbol(str)
+        case str
+        when '+'
+          :must
+        when '-'
+          :must_not
+        when nil
+          :should
+        else
+          raise "Unknown operator: #{str}"
+        end
+      end
+    end
+  end
+
+  class TermClause
+    attr_reader :prefix, :operator, :term
+
+    def initialize(prefix, operator, term)
+      @prefix = prefix
+      @operator = Operator.symbol(operator)
+      @term = term
+    end
+  end
+
+  class PhraseClause
+    attr_reader :prefix, :operator, :phrase
+
+    def initialize(prefix, operator, phrase)
+      @prefix = prefix
+      @operator = Operator.symbol(operator)
+      @phrase = phrase
+    end
+  end
+
+  rule(clause: subtree(:clause)) do
+    prefix   = clause[:prefix][:term].to_s if clause[:prefix]
+    operator = clause[:operator]&.to_s
+
+    if clause[:term]
+      TermClause.new(prefix, operator, clause[:term].to_s)
+    elsif clause[:phrase]
+      PhraseClause.new(prefix, operator, clause[:phrase].map { |p| p[:term].to_s }.join(' '))
+    else
+      raise "Unexpected clause type: #{clause}"
+    end
+  end
+
+  rule(query: sequence(:clauses)) { Query.new(clauses) }
+end
diff --git a/app/services/search_service.rb b/app/services/search_service.rb
index e0da61dac..769d1ac7a 100644
--- a/app/services/search_service.rb
+++ b/app/services/search_service.rb
@@ -33,8 +33,7 @@ class SearchService < BaseService
   end
 
   def perform_statuses_search!
-    definition = StatusesIndex.filter(term: { searchable_by: @account.id })
-                              .query(multi_match: { type: 'most_fields', query: @query, operator: 'and', fields: %w(text text.stemmed) })
+    definition = parsed_query.apply(StatusesIndex.filter(term: { searchable_by: @account.id }))
 
     if @options[:account_id].present?
       definition = definition.filter(term: { account_id: @options[:account_id] })
@@ -70,7 +69,7 @@ class SearchService < BaseService
   end
 
   def url_query?
-    @options[:type].blank? && @query =~ /\Ahttps?:\/\//
+    @resolve && @options[:type].blank? && @query =~ /\Ahttps?:\/\//
   end
 
   def url_resource_results
@@ -120,4 +119,8 @@ class SearchService < BaseService
       domain_blocking_by_domain: Account.domain_blocking_map_by_domain(domains, account.id),
     }
   end
+
+  def parsed_query
+    SearchQueryTransformer.new.apply(SearchQueryParser.new.parse(@query))
+  end
 end
diff --git a/spec/services/search_service_spec.rb b/spec/services/search_service_spec.rb
index d064cd9b8..ade306ed2 100644
--- a/spec/services/search_service_spec.rb
+++ b/spec/services/search_service_spec.rb
@@ -27,7 +27,7 @@ describe SearchService, type: :service do
         it 'returns the empty results' do
           service = double(call: nil)
           allow(ResolveURLService).to receive(:new).and_return(service)
-          results = subject.call(@query, nil, 10)
+          results = subject.call(@query, nil, 10, resolve: true)
 
           expect(service).to have_received(:call).with(@query, on_behalf_of: nil)
           expect(results).to eq empty_results
@@ -40,7 +40,7 @@ describe SearchService, type: :service do
           service = double(call: account)
           allow(ResolveURLService).to receive(:new).and_return(service)
 
-          results = subject.call(@query, nil, 10)
+          results = subject.call(@query, nil, 10, resolve: true)
           expect(service).to have_received(:call).with(@query, on_behalf_of: nil)
           expect(results).to eq empty_results.merge(accounts: [account])
         end
@@ -52,7 +52,7 @@ describe SearchService, type: :service do
           service = double(call: status)
           allow(ResolveURLService).to receive(:new).and_return(service)
 
-          results = subject.call(@query, nil, 10)
+          results = subject.call(@query, nil, 10, resolve: true)
           expect(service).to have_received(:call).with(@query, on_behalf_of: nil)
           expect(results).to eq empty_results.merge(statuses: [status])
         end
-- 
cgit 


From d6ada2eb30a2e73b71d8ef037d3764ff006fb745 Mon Sep 17 00:00:00 2001
From: ysksn <bluewhale1982@gmail.com>
Date: Sat, 27 Jul 2019 17:24:26 +0900
Subject: Implement pending tests (#11415)

---
 spec/lib/spam_check_spec.rb   | 28 +++++++++++++++++++++++++++-
 spec/models/poll_vote_spec.rb | 10 +++++++++-
 2 files changed, 36 insertions(+), 2 deletions(-)

(limited to 'spec')

diff --git a/spec/lib/spam_check_spec.rb b/spec/lib/spam_check_spec.rb
index c722dc642..9e0989216 100644
--- a/spec/lib/spam_check_spec.rb
+++ b/spec/lib/spam_check_spec.rb
@@ -1,3 +1,5 @@
+# frozen_string_literal: true
+
 require 'rails_helper'
 
 RSpec.describe SpamCheck do
@@ -133,7 +135,31 @@ RSpec.describe SpamCheck do
   end
 
   describe '#remember!' do
-    pending
+    let(:status) { status_with_html('@alice') }
+    let(:spam_check) { described_class.new(status) }
+    let(:redis_key) { spam_check.send(:redis_key) }
+
+    it 'remembers' do
+      expect do
+        spam_check.remember!
+      end.to change { Redis.current.exists(redis_key) }.from(false).to(true)
+    end
+  end
+
+  describe '#reset!' do
+    let(:status) { status_with_html('@alice') }
+    let(:spam_check) { described_class.new(status) }
+    let(:redis_key) { spam_check.send(:redis_key) }
+
+    before do
+      spam_check.remember!
+    end
+
+    it 'resets' do
+      expect do
+        spam_check.reset!
+      end.to change { Redis.current.exists(redis_key) }.from(true).to(false)
+    end
   end
 
   describe '#flag!' do
diff --git a/spec/models/poll_vote_spec.rb b/spec/models/poll_vote_spec.rb
index 354afd535..563f34699 100644
--- a/spec/models/poll_vote_spec.rb
+++ b/spec/models/poll_vote_spec.rb
@@ -1,5 +1,13 @@
+# frozen_string_literal: true
+
 require 'rails_helper'
 
 RSpec.describe PollVote, type: :model do
-  pending "add some examples to (or delete) #{__FILE__}"
+  describe '#object_type' do
+    let(:poll_vote) { Fabricate.build(:poll_vote) }
+
+    it 'returns :vote' do
+      expect(poll_vote.object_type).to eq :vote
+    end
+  end
 end
-- 
cgit 


From 75f7f9930eb2a6f5c4041ec44fe0aa795c9ec449 Mon Sep 17 00:00:00 2001
From: Eugen Rochko <eugen@zeonfederated.com>
Date: Sun, 28 Jul 2019 17:30:12 +0200
Subject: Remove conversation URI (#11423)

It is not part of ActivityPub and will free up a lot of space
---
 app/lib/activitypub/activity/create.rb                    | 11 -----------
 app/models/conversation.rb                                |  7 -------
 app/serializers/activitypub/note_serializer.rb            | 15 ++-------------
 .../20190519130537_remove_boosts_widening_audience.rb     |  2 ++
 .../20190728084117_remove_uri_from_conversations.rb       | 12 ++++++++++++
 db/schema.rb                                              |  4 +---
 spec/models/conversation_spec.rb                          |  8 --------
 7 files changed, 17 insertions(+), 42 deletions(-)
 create mode 100644 db/post_migrate/20190728084117_remove_uri_from_conversations.rb

(limited to 'spec')

diff --git a/app/lib/activitypub/activity/create.rb b/app/lib/activitypub/activity/create.rb
index 000b77df5..eb7b09e44 100644
--- a/app/lib/activitypub/activity/create.rb
+++ b/app/lib/activitypub/activity/create.rb
@@ -67,7 +67,6 @@ class ActivityPub::Activity::Create < ActivityPub::Activity
         sensitive: @object['sensitive'] || false,
         visibility: visibility_from_audience,
         thread: replied_to_status,
-        conversation: conversation_from_uri(@object['conversation']),
         media_attachment_ids: process_attachments.take(4).map(&:id),
         poll: process_poll,
       }
@@ -262,16 +261,6 @@ class ActivityPub::Activity::Create < ActivityPub::Activity
     ActivityPub::FetchRepliesWorker.perform_async(status.id, uri) unless uri.nil?
   end
 
-  def conversation_from_uri(uri)
-    return nil if uri.nil?
-    return Conversation.find_by(id: OStatus::TagManager.instance.unique_tag_to_local_id(uri, 'Conversation')) if OStatus::TagManager.instance.local_id?(uri)
-    begin
-      Conversation.find_or_create_by!(uri: uri)
-    rescue ActiveRecord::RecordInvalid, ActiveRecord::RecordNotUnique
-      retry
-    end
-  end
-
   def visibility_from_audience
     if equals_or_includes?(@object['to'], ActivityPub::TagManager::COLLECTIONS[:public])
       :public
diff --git a/app/models/conversation.rb b/app/models/conversation.rb
index 4dfaea889..37d233f32 100644
--- a/app/models/conversation.rb
+++ b/app/models/conversation.rb
@@ -4,17 +4,10 @@
 # Table name: conversations
 #
 #  id         :bigint(8)        not null, primary key
-#  uri        :string
 #  created_at :datetime         not null
 #  updated_at :datetime         not null
 #
 
 class Conversation < ApplicationRecord
-  validates :uri, uniqueness: true, if: :uri?
-
   has_many :statuses
-
-  def local?
-    uri.nil?
-  end
 end
diff --git a/app/serializers/activitypub/note_serializer.rb b/app/serializers/activitypub/note_serializer.rb
index 67f596e78..88b50b3ea 100644
--- a/app/serializers/activitypub/note_serializer.rb
+++ b/app/serializers/activitypub/note_serializer.rb
@@ -1,14 +1,13 @@
 # frozen_string_literal: true
 
 class ActivityPub::NoteSerializer < ActivityPub::Serializer
-  context_extensions :atom_uri, :conversation, :sensitive,
+  context_extensions :atom_uri, :sensitive,
                      :hashtag, :emoji, :focal_point, :blurhash
 
   attributes :id, :type, :summary,
              :in_reply_to, :published, :url,
              :attributed_to, :to, :cc, :sensitive,
-             :atom_uri, :in_reply_to_atom_uri,
-             :conversation
+             :atom_uri, :in_reply_to_atom_uri
 
   attribute :content
   attribute :content_map, if: :language?
@@ -110,16 +109,6 @@ class ActivityPub::NoteSerializer < ActivityPub::Serializer
     OStatus::TagManager.instance.uri_for(object.thread)
   end
 
-  def conversation
-    return if object.conversation.nil?
-
-    if object.conversation.uri?
-      object.conversation.uri
-    else
-      OStatus::TagManager.instance.unique_tag(object.conversation.created_at, object.conversation.id, 'Conversation')
-    end
-  end
-
   def local?
     object.account.local?
   end
diff --git a/db/post_migrate/20190519130537_remove_boosts_widening_audience.rb b/db/post_migrate/20190519130537_remove_boosts_widening_audience.rb
index d2d924239..b89efd989 100644
--- a/db/post_migrate/20190519130537_remove_boosts_widening_audience.rb
+++ b/db/post_migrate/20190519130537_remove_boosts_widening_audience.rb
@@ -1,3 +1,5 @@
+# frozen_string_literal: true
+
 class RemoveBoostsWideningAudience < ActiveRecord::Migration[5.2]
   disable_ddl_transaction!
 
diff --git a/db/post_migrate/20190728084117_remove_uri_from_conversations.rb b/db/post_migrate/20190728084117_remove_uri_from_conversations.rb
new file mode 100644
index 000000000..b2b157ef7
--- /dev/null
+++ b/db/post_migrate/20190728084117_remove_uri_from_conversations.rb
@@ -0,0 +1,12 @@
+# frozen_string_literal: true
+
+class RemoveUriFromConversations < ActiveRecord::Migration[5.2]
+  def up
+    safety_assured { remove_column :conversations, :uri, :string }
+  end
+
+  def down
+    add_column :conversations, :uri, :string
+    add_index :conversations, :uri, unique: true
+  end
+end
diff --git a/db/schema.rb b/db/schema.rb
index 1847305c7..a722bad36 100644
--- a/db/schema.rb
+++ b/db/schema.rb
@@ -10,7 +10,7 @@
 #
 # It's strongly recommended that you check this file into your version control system.
 
-ActiveRecord::Schema.define(version: 2019_07_26_175042) do
+ActiveRecord::Schema.define(version: 2019_07_28_084117) do
 
   # These are extensions that must be enabled in order to support this database
   enable_extension "plpgsql"
@@ -203,10 +203,8 @@ ActiveRecord::Schema.define(version: 2019_07_26_175042) do
   end
 
   create_table "conversations", force: :cascade do |t|
-    t.string "uri"
     t.datetime "created_at", null: false
     t.datetime "updated_at", null: false
-    t.index ["uri"], name: "index_conversations_on_uri", unique: true
   end
 
   create_table "custom_emoji_categories", force: :cascade do |t|
diff --git a/spec/models/conversation_spec.rb b/spec/models/conversation_spec.rb
index 8b5e4fdaf..b91d36169 100644
--- a/spec/models/conversation_spec.rb
+++ b/spec/models/conversation_spec.rb
@@ -1,13 +1,5 @@
 require 'rails_helper'
 
 RSpec.describe Conversation, type: :model do
-  describe '#local?' do
-    it 'returns true when URI is nil' do
-      expect(Fabricate(:conversation).local?).to be true
-    end
 
-    it 'returns false when URI is not nil' do
-      expect(Fabricate(:conversation, uri: 'abc').local?).to be false
-    end
-  end
 end
-- 
cgit 


From b9b0313c78ca4efcc8cda089c652be0b333aa16a Mon Sep 17 00:00:00 2001
From: Eugen Rochko <eugen@zeonfederated.com>
Date: Sun, 28 Jul 2019 17:47:37 +0200
Subject: Revert "Remove conversation URI (#11423)" (#11424)

This reverts commit 75f7f9930eb2a6f5c4041ec44fe0aa795c9ec449.
---
 app/lib/activitypub/activity/create.rb                    | 11 +++++++++++
 app/models/conversation.rb                                |  7 +++++++
 app/serializers/activitypub/note_serializer.rb            | 15 +++++++++++++--
 .../20190519130537_remove_boosts_widening_audience.rb     |  2 --
 .../20190728084117_remove_uri_from_conversations.rb       | 12 ------------
 db/schema.rb                                              |  4 +++-
 spec/models/conversation_spec.rb                          |  8 ++++++++
 7 files changed, 42 insertions(+), 17 deletions(-)
 delete mode 100644 db/post_migrate/20190728084117_remove_uri_from_conversations.rb

(limited to 'spec')

diff --git a/app/lib/activitypub/activity/create.rb b/app/lib/activitypub/activity/create.rb
index eb7b09e44..000b77df5 100644
--- a/app/lib/activitypub/activity/create.rb
+++ b/app/lib/activitypub/activity/create.rb
@@ -67,6 +67,7 @@ class ActivityPub::Activity::Create < ActivityPub::Activity
         sensitive: @object['sensitive'] || false,
         visibility: visibility_from_audience,
         thread: replied_to_status,
+        conversation: conversation_from_uri(@object['conversation']),
         media_attachment_ids: process_attachments.take(4).map(&:id),
         poll: process_poll,
       }
@@ -261,6 +262,16 @@ class ActivityPub::Activity::Create < ActivityPub::Activity
     ActivityPub::FetchRepliesWorker.perform_async(status.id, uri) unless uri.nil?
   end
 
+  def conversation_from_uri(uri)
+    return nil if uri.nil?
+    return Conversation.find_by(id: OStatus::TagManager.instance.unique_tag_to_local_id(uri, 'Conversation')) if OStatus::TagManager.instance.local_id?(uri)
+    begin
+      Conversation.find_or_create_by!(uri: uri)
+    rescue ActiveRecord::RecordInvalid, ActiveRecord::RecordNotUnique
+      retry
+    end
+  end
+
   def visibility_from_audience
     if equals_or_includes?(@object['to'], ActivityPub::TagManager::COLLECTIONS[:public])
       :public
diff --git a/app/models/conversation.rb b/app/models/conversation.rb
index 37d233f32..4dfaea889 100644
--- a/app/models/conversation.rb
+++ b/app/models/conversation.rb
@@ -4,10 +4,17 @@
 # Table name: conversations
 #
 #  id         :bigint(8)        not null, primary key
+#  uri        :string
 #  created_at :datetime         not null
 #  updated_at :datetime         not null
 #
 
 class Conversation < ApplicationRecord
+  validates :uri, uniqueness: true, if: :uri?
+
   has_many :statuses
+
+  def local?
+    uri.nil?
+  end
 end
diff --git a/app/serializers/activitypub/note_serializer.rb b/app/serializers/activitypub/note_serializer.rb
index 88b50b3ea..67f596e78 100644
--- a/app/serializers/activitypub/note_serializer.rb
+++ b/app/serializers/activitypub/note_serializer.rb
@@ -1,13 +1,14 @@
 # frozen_string_literal: true
 
 class ActivityPub::NoteSerializer < ActivityPub::Serializer
-  context_extensions :atom_uri, :sensitive,
+  context_extensions :atom_uri, :conversation, :sensitive,
                      :hashtag, :emoji, :focal_point, :blurhash
 
   attributes :id, :type, :summary,
              :in_reply_to, :published, :url,
              :attributed_to, :to, :cc, :sensitive,
-             :atom_uri, :in_reply_to_atom_uri
+             :atom_uri, :in_reply_to_atom_uri,
+             :conversation
 
   attribute :content
   attribute :content_map, if: :language?
@@ -109,6 +110,16 @@ class ActivityPub::NoteSerializer < ActivityPub::Serializer
     OStatus::TagManager.instance.uri_for(object.thread)
   end
 
+  def conversation
+    return if object.conversation.nil?
+
+    if object.conversation.uri?
+      object.conversation.uri
+    else
+      OStatus::TagManager.instance.unique_tag(object.conversation.created_at, object.conversation.id, 'Conversation')
+    end
+  end
+
   def local?
     object.account.local?
   end
diff --git a/db/post_migrate/20190519130537_remove_boosts_widening_audience.rb b/db/post_migrate/20190519130537_remove_boosts_widening_audience.rb
index b89efd989..d2d924239 100644
--- a/db/post_migrate/20190519130537_remove_boosts_widening_audience.rb
+++ b/db/post_migrate/20190519130537_remove_boosts_widening_audience.rb
@@ -1,5 +1,3 @@
-# frozen_string_literal: true
-
 class RemoveBoostsWideningAudience < ActiveRecord::Migration[5.2]
   disable_ddl_transaction!
 
diff --git a/db/post_migrate/20190728084117_remove_uri_from_conversations.rb b/db/post_migrate/20190728084117_remove_uri_from_conversations.rb
deleted file mode 100644
index b2b157ef7..000000000
--- a/db/post_migrate/20190728084117_remove_uri_from_conversations.rb
+++ /dev/null
@@ -1,12 +0,0 @@
-# frozen_string_literal: true
-
-class RemoveUriFromConversations < ActiveRecord::Migration[5.2]
-  def up
-    safety_assured { remove_column :conversations, :uri, :string }
-  end
-
-  def down
-    add_column :conversations, :uri, :string
-    add_index :conversations, :uri, unique: true
-  end
-end
diff --git a/db/schema.rb b/db/schema.rb
index a722bad36..1847305c7 100644
--- a/db/schema.rb
+++ b/db/schema.rb
@@ -10,7 +10,7 @@
 #
 # It's strongly recommended that you check this file into your version control system.
 
-ActiveRecord::Schema.define(version: 2019_07_28_084117) do
+ActiveRecord::Schema.define(version: 2019_07_26_175042) do
 
   # These are extensions that must be enabled in order to support this database
   enable_extension "plpgsql"
@@ -203,8 +203,10 @@ ActiveRecord::Schema.define(version: 2019_07_28_084117) do
   end
 
   create_table "conversations", force: :cascade do |t|
+    t.string "uri"
     t.datetime "created_at", null: false
     t.datetime "updated_at", null: false
+    t.index ["uri"], name: "index_conversations_on_uri", unique: true
   end
 
   create_table "custom_emoji_categories", force: :cascade do |t|
diff --git a/spec/models/conversation_spec.rb b/spec/models/conversation_spec.rb
index b91d36169..8b5e4fdaf 100644
--- a/spec/models/conversation_spec.rb
+++ b/spec/models/conversation_spec.rb
@@ -1,5 +1,13 @@
 require 'rails_helper'
 
 RSpec.describe Conversation, type: :model do
+  describe '#local?' do
+    it 'returns true when URI is nil' do
+      expect(Fabricate(:conversation).local?).to be true
+    end
 
+    it 'returns false when URI is not nil' do
+      expect(Fabricate(:conversation, uri: 'abc').local?).to be false
+    end
+  end
 end
-- 
cgit 


From e136112ab76a37cbde5c1bcfeb562c8e0a5b5116 Mon Sep 17 00:00:00 2001
From: Eugen Rochko <eugen@zeonfederated.com>
Date: Mon, 29 Jul 2019 20:40:21 +0200
Subject: Fix tag normalization and migration not removing duplicate tags
 (#11441)

Fix #11428
---
 app/models/tag.rb                                  |  8 ++---
 ...726175042_add_case_insensitive_index_to_tags.rb | 13 +++++++++
 spec/models/tag_spec.rb                            | 34 ++++++++++++++++++++++
 3 files changed, 51 insertions(+), 4 deletions(-)

(limited to 'spec')

diff --git a/app/models/tag.rb b/app/models/tag.rb
index 972242064..46e3a3ec0 100644
--- a/app/models/tag.rb
+++ b/app/models/tag.rb
@@ -65,7 +65,7 @@ class Tag < ApplicationRecord
 
   class << self
     def find_or_create_by_names(name_or_names)
-      Array(name_or_names).map(&method(:normalize)).uniq.map do |normalized_name|
+      Array(name_or_names).map(&method(:normalize)).uniq { |str| str.mb_chars.downcase.to_s }.map do |normalized_name|
         tag = matching_name(normalized_name).first || create(name: normalized_name)
 
         yield tag if block_given?
@@ -77,7 +77,7 @@ class Tag < ApplicationRecord
     def search_for(term, limit = 5, offset = 0)
       pattern = sanitize_sql_like(normalize(term.strip)) + '%'
 
-      Tag.where(arel_table[:name].lower.matches(pattern.downcase))
+      Tag.where(arel_table[:name].lower.matches(pattern.mb_chars.downcase.to_s))
          .order(:name)
          .limit(limit)
          .offset(offset)
@@ -92,7 +92,7 @@ class Tag < ApplicationRecord
     end
 
     def matching_name(name_or_names)
-      names = Array(name_or_names).map { |name| normalize(name).downcase }
+      names = Array(name_or_names).map { |name| normalize(name).mb_chars.downcase.to_s }
 
       if names.size == 1
         where(arel_table[:name].lower.eq(names.first))
@@ -104,7 +104,7 @@ class Tag < ApplicationRecord
     private
 
     def normalize(str)
-      str.gsub(/\A#/, '').mb_chars.to_s
+      str.gsub(/\A#/, '')
     end
   end
 
diff --git a/db/migrate/20190726175042_add_case_insensitive_index_to_tags.rb b/db/migrate/20190726175042_add_case_insensitive_index_to_tags.rb
index 6fa8c0ec4..057fc86ba 100644
--- a/db/migrate/20190726175042_add_case_insensitive_index_to_tags.rb
+++ b/db/migrate/20190726175042_add_case_insensitive_index_to_tags.rb
@@ -2,6 +2,19 @@ class AddCaseInsensitiveIndexToTags < ActiveRecord::Migration[5.2]
   disable_ddl_transaction!
 
   def up
+    Tag.connection.select_all('SELECT string_agg(id::text, \',\') AS ids FROM tags GROUP BY lower(name) HAVING count(*) > 1').to_hash.each do |row|
+      canonical_tag_id  = row['ids'].split(',').first
+      redundant_tag_ids = row['ids'].split(',')[1..-1]
+
+      safety_assured do
+        execute "UPDATE accounts_tags AS t0 SET tag_id = #{canonical_tag_id} WHERE tag_id IN (#{redundant_tag_ids.join(', ')}) AND NOT EXISTS (SELECT t1.tag_id FROM accounts_tags AS t1 WHERE t1.tag_id = #{canonical_tag_id} AND t1.account_id = t0.account_id)"
+        execute "UPDATE statuses_tags AS t0 SET tag_id = #{canonical_tag_id} WHERE tag_id IN (#{redundant_tag_ids.join(', ')}) AND NOT EXISTS (SELECT t1.tag_id FROM statuses_tags AS t1 WHERE t1.tag_id = #{canonical_tag_id} AND t1.status_id = t0.status_id)"
+        execute "UPDATE featured_tags AS t0 SET tag_id = #{canonical_tag_id} WHERE tag_id IN (#{redundant_tag_ids.join(', ')})  AND NOT EXISTS (SELECT t1.tag_id FROM featured_tags AS t1 WHERE t1.tag_id = #{canonical_tag_id} AND t1.account_id = t0.account_id)"
+      end
+
+      Tag.where(id: redundant_tag_ids).in_batches.delete_all
+    end
+
     safety_assured { execute 'CREATE UNIQUE INDEX CONCURRENTLY index_tags_on_name_lower ON tags (lower(name))' }
     remove_index :tags, name: 'index_tags_on_name'
     remove_index :tags, name: 'hashtag_search_index'
diff --git a/spec/models/tag_spec.rb b/spec/models/tag_spec.rb
index 9a30ceaa5..5f07fd618 100644
--- a/spec/models/tag_spec.rb
+++ b/spec/models/tag_spec.rb
@@ -82,6 +82,40 @@ RSpec.describe Tag, type: :model do
     end
   end
 
+  describe '.find_normalized' do
+    it 'returns tag for a multibyte case-insensitive name' do
+      upcase_string   = 'abcABCａｂｃＡＢＣやゆよ'
+      downcase_string = 'abcabcａｂｃａｂｃやゆよ';
+
+      tag = Fabricate(:tag, name: downcase_string)
+      expect(Tag.find_normalized(upcase_string)).to eq tag
+    end
+  end
+
+  describe '.matching_name' do
+    it 'returns tags for multibyte case-insensitive names' do
+      upcase_string   = 'abcABCａｂｃＡＢＣやゆよ'
+      downcase_string = 'abcabcａｂｃａｂｃやゆよ';
+
+      tag = Fabricate(:tag, name: downcase_string)
+      expect(Tag.matching_name(upcase_string)).to eq [tag]
+    end
+  end
+
+  describe '.find_or_create_by_names' do
+    it 'runs a passed block once per tag regardless of duplicates' do
+      upcase_string   = 'abcABCａｂｃＡＢＣやゆよ'
+      downcase_string = 'abcabcａｂｃａｂｃやゆよ';
+      count           = 0
+
+      Tag.find_or_create_by_names([upcase_string, downcase_string]) do |tag|
+        count += 1
+      end
+
+      expect(count).to eq 1
+    end
+  end
+
   describe '.search_for' do
     it 'finds tag records with matching names' do
       tag = Fabricate(:tag, name: "match")
-- 
cgit 


From 24552b5160a5090e7d6056fb69a209aa48fe4fce Mon Sep 17 00:00:00 2001
From: Eugen Rochko <eugen@zeonfederated.com>
Date: Tue, 30 Jul 2019 11:10:46 +0200
Subject: Add whitelist mode (#11291)

---
 app/controllers/about_controller.rb                |  5 +++
 app/controllers/activitypub/base_controller.rb     |  2 ++
 app/controllers/activitypub/inboxes_controller.rb  |  2 +-
 app/controllers/admin/domain_allows_controller.rb  | 40 ++++++++++++++++++++++
 app/controllers/admin/instances_controller.rb      | 28 +++++++++++++--
 app/controllers/api/base_controller.rb             |  9 +++++
 app/controllers/api/v1/accounts_controller.rb      |  2 ++
 app/controllers/api/v1/apps_controller.rb          |  2 ++
 .../api/v1/instances/activity_controller.rb        |  3 +-
 .../api/v1/instances/peers_controller.rb           |  3 +-
 app/controllers/api/v1/instances_controller.rb     |  1 +
 app/controllers/application_controller.rb          |  4 ++-
 app/controllers/concerns/account_owned_concern.rb  |  1 +
 app/controllers/directories_controller.rb          |  5 +--
 app/controllers/home_controller.rb                 |  2 +-
 app/controllers/media_controller.rb                |  1 +
 app/controllers/media_proxy_controller.rb          |  2 ++
 app/controllers/public_timelines_controller.rb     |  5 +--
 app/controllers/remote_interaction_controller.rb   |  1 +
 app/controllers/tags_controller.rb                 |  1 +
 app/helpers/domain_control_helper.rb               | 10 +++++-
 app/models/domain_allow.rb                         | 33 ++++++++++++++++++
 app/models/instance.rb                             |  3 +-
 app/models/instance_filter.rb                      |  4 +++
 app/policies/domain_allow_policy.rb                | 11 ++++++
 app/services/concerns/payloadable.rb               |  2 +-
 app/services/unallow_domain_service.rb             | 11 ++++++
 app/views/admin/domain_allows/new.html.haml        | 14 ++++++++
 app/views/admin/instances/index.html.haml          | 35 ++++++++++++-------
 app/views/admin/instances/show.html.haml           |  4 ++-
 app/views/admin/settings/edit.html.haml            | 28 ++++++++-------
 app/views/auth/registrations/new.html.haml         |  2 +-
 app/views/layouts/public.html.haml                 |  9 +++--
 config/initializers/2_whitelist_mode.rb            |  5 +++
 config/locales/en.yml                              |  7 ++++
 config/locales/simple_form.en.yml                  |  2 ++
 config/navigation.rb                               |  2 +-
 config/routes.rb                                   |  1 +
 db/migrate/20190705002136_create_domain_allows.rb  |  9 +++++
 db/schema.rb                                       |  9 ++++-
 lib/mastodon/domains_cli.rb                        | 22 ++++++++++--
 spec/fabricators/domain_allow_fabricator.rb        |  3 ++
 spec/models/domain_allow_spec.rb                   |  5 +++
 streaming/index.js                                 |  5 +--
 44 files changed, 302 insertions(+), 53 deletions(-)
 create mode 100644 app/controllers/admin/domain_allows_controller.rb
 create mode 100644 app/models/domain_allow.rb
 create mode 100644 app/policies/domain_allow_policy.rb
 create mode 100644 app/services/unallow_domain_service.rb
 create mode 100644 app/views/admin/domain_allows/new.html.haml
 create mode 100644 config/initializers/2_whitelist_mode.rb
 create mode 100644 db/migrate/20190705002136_create_domain_allows.rb
 create mode 100644 spec/fabricators/domain_allow_fabricator.rb
 create mode 100644 spec/models/domain_allow_spec.rb

(limited to 'spec')

diff --git a/app/controllers/about_controller.rb b/app/controllers/about_controller.rb
index 31cf17710..d276e8fe5 100644
--- a/app/controllers/about_controller.rb
+++ b/app/controllers/about_controller.rb
@@ -3,6 +3,7 @@
 class AboutController < ApplicationController
   layout 'public'
 
+  before_action :require_open_federation!, only: [:show, :more]
   before_action :set_body_classes, only: :show
   before_action :set_instance_presenter
   before_action :set_expires_in
@@ -19,6 +20,10 @@ class AboutController < ApplicationController
 
   private
 
+  def require_open_federation!
+    not_found if whitelist_mode?
+  end
+
   def new_user
     User.new.tap do |user|
       user.build_account
diff --git a/app/controllers/activitypub/base_controller.rb b/app/controllers/activitypub/base_controller.rb
index a3b5c4dfa..0c2591e97 100644
--- a/app/controllers/activitypub/base_controller.rb
+++ b/app/controllers/activitypub/base_controller.rb
@@ -1,6 +1,8 @@
 # frozen_string_literal: true
 
 class ActivityPub::BaseController < Api::BaseController
+  skip_before_action :require_authenticated_user!
+
   private
 
   def set_cache_headers
diff --git a/app/controllers/activitypub/inboxes_controller.rb b/app/controllers/activitypub/inboxes_controller.rb
index 7cfd9a25e..bcfc1e6d4 100644
--- a/app/controllers/activitypub/inboxes_controller.rb
+++ b/app/controllers/activitypub/inboxes_controller.rb
@@ -1,6 +1,6 @@
 # frozen_string_literal: true
 
-class ActivityPub::InboxesController < Api::BaseController
+class ActivityPub::InboxesController < ActivityPub::BaseController
   include SignatureVerification
   include JsonLdHelper
   include AccountOwnedConcern
diff --git a/app/controllers/admin/domain_allows_controller.rb b/app/controllers/admin/domain_allows_controller.rb
new file mode 100644
index 000000000..31be1978b
--- /dev/null
+++ b/app/controllers/admin/domain_allows_controller.rb
@@ -0,0 +1,40 @@
+# frozen_string_literal: true
+
+class Admin::DomainAllowsController < Admin::BaseController
+  before_action :set_domain_allow, only: [:destroy]
+
+  def new
+    authorize :domain_allow, :create?
+
+    @domain_allow = DomainAllow.new(domain: params[:_domain])
+  end
+
+  def create
+    authorize :domain_allow, :create?
+
+    @domain_allow = DomainAllow.new(resource_params)
+
+    if @domain_allow.save
+      log_action :create, @domain_allow
+      redirect_to admin_instances_path, notice: I18n.t('admin.domain_allows.created_msg')
+    else
+      render :new
+    end
+  end
+
+  def destroy
+    authorize @domain_allow, :destroy?
+    UnallowDomainService.new.call(@domain_allow)
+    redirect_to admin_instances_path, notice: I18n.t('admin.domain_allows.destroyed_msg')
+  end
+
+  private
+
+  def set_domain_allow
+    @domain_allow = DomainAllow.find(params[:id])
+  end
+
+  def resource_params
+    params.require(:domain_allow).permit(:domain)
+  end
+end
diff --git a/app/controllers/admin/instances_controller.rb b/app/controllers/admin/instances_controller.rb
index 7888e844f..d4f201807 100644
--- a/app/controllers/admin/instances_controller.rb
+++ b/app/controllers/admin/instances_controller.rb
@@ -2,6 +2,10 @@
 
 module Admin
   class InstancesController < BaseController
+    before_action :set_domain_block, only: :show
+    before_action :set_domain_allow, only: :show
+    before_action :set_instance, only: :show
+
     def index
       authorize :instance, :index?
 
@@ -11,20 +15,38 @@ module Admin
     def show
       authorize :instance, :show?
 
-      @instance        = Instance.new(Account.by_domain_accounts.find_by(domain: params[:id]) || DomainBlock.find_by!(domain: params[:id]))
       @following_count = Follow.where(account: Account.where(domain: params[:id])).count
       @followers_count = Follow.where(target_account: Account.where(domain: params[:id])).count
       @reports_count   = Report.where(target_account: Account.where(domain: params[:id])).count
       @blocks_count    = Block.where(target_account: Account.where(domain: params[:id])).count
       @available       = DeliveryFailureTracker.available?(Account.select(:shared_inbox_url).where(domain: params[:id]).first&.shared_inbox_url)
       @media_storage   = MediaAttachment.where(account: Account.where(domain: params[:id])).sum(:file_file_size)
-      @domain_block    = DomainBlock.rule_for(params[:id])
     end
 
     private
 
+    def set_domain_block
+      @domain_block = DomainBlock.rule_for(params[:id])
+    end
+
+    def set_domain_allow
+      @domain_allow = DomainAllow.rule_for(params[:id])
+    end
+
+    def set_instance
+      resource   = Account.by_domain_accounts.find_by(domain: params[:id])
+      resource ||= @domain_block
+      resource ||= @domain_allow
+
+      if resource
+        @instance = Instance.new(resource)
+      else
+        not_found
+      end
+    end
+
     def filtered_instances
-      InstanceFilter.new(filter_params).results
+      InstanceFilter.new(whitelist_mode? ? { allowed: true } : filter_params).results
     end
 
     def paginated_instances
diff --git a/app/controllers/api/base_controller.rb b/app/controllers/api/base_controller.rb
index 6f33a1ea9..109e38ffa 100644
--- a/app/controllers/api/base_controller.rb
+++ b/app/controllers/api/base_controller.rb
@@ -9,6 +9,7 @@ class Api::BaseController < ApplicationController
   skip_before_action :store_current_location
   skip_before_action :require_functional!
 
+  before_action :require_authenticated_user!, if: :disallow_unauthenticated_api_access?
   before_action :set_cache_headers
 
   protect_from_forgery with: :null_session
@@ -69,6 +70,10 @@ class Api::BaseController < ApplicationController
     nil
   end
 
+  def require_authenticated_user!
+    render json: { error: 'This API requires an authenticated user' }, status: 401 unless current_user
+  end
+
   def require_user!
     if !current_user
       render json: { error: 'This method requires an authenticated user' }, status: 422
@@ -94,4 +99,8 @@ class Api::BaseController < ApplicationController
   def set_cache_headers
     response.headers['Cache-Control'] = 'no-cache, no-store, max-age=0, must-revalidate'
   end
+
+  def disallow_unauthenticated_api_access?
+    authorized_fetch_mode?
+  end
 end
diff --git a/app/controllers/api/v1/accounts_controller.rb b/app/controllers/api/v1/accounts_controller.rb
index b0c62778e..b306e8e8c 100644
--- a/app/controllers/api/v1/accounts_controller.rb
+++ b/app/controllers/api/v1/accounts_controller.rb
@@ -12,6 +12,8 @@ class Api::V1::AccountsController < Api::BaseController
   before_action :check_account_suspension, only: [:show]
   before_action :check_enabled_registrations, only: [:create]
 
+  skip_before_action :require_authenticated_user!, only: :create
+
   respond_to :json
 
   def show
diff --git a/app/controllers/api/v1/apps_controller.rb b/app/controllers/api/v1/apps_controller.rb
index e9f7a7291..97177547a 100644
--- a/app/controllers/api/v1/apps_controller.rb
+++ b/app/controllers/api/v1/apps_controller.rb
@@ -1,6 +1,8 @@
 # frozen_string_literal: true
 
 class Api::V1::AppsController < Api::BaseController
+  skip_before_action :require_authenticated_user!
+
   def create
     @app = Doorkeeper::Application.create!(application_options)
     render json: @app, serializer: REST::ApplicationSerializer
diff --git a/app/controllers/api/v1/instances/activity_controller.rb b/app/controllers/api/v1/instances/activity_controller.rb
index d0080c5c2..4fb5a69d8 100644
--- a/app/controllers/api/v1/instances/activity_controller.rb
+++ b/app/controllers/api/v1/instances/activity_controller.rb
@@ -2,6 +2,7 @@
 
 class Api::V1::Instances::ActivityController < Api::BaseController
   before_action :require_enabled_api!
+
   skip_before_action :set_cache_headers
 
   respond_to :json
@@ -33,6 +34,6 @@ class Api::V1::Instances::ActivityController < Api::BaseController
   end
 
   def require_enabled_api!
-    head 404 unless Setting.activity_api_enabled
+    head 404 unless Setting.activity_api_enabled && !whitelist_mode?
   end
 end
diff --git a/app/controllers/api/v1/instances/peers_controller.rb b/app/controllers/api/v1/instances/peers_controller.rb
index 450e6502f..75c3cb4ba 100644
--- a/app/controllers/api/v1/instances/peers_controller.rb
+++ b/app/controllers/api/v1/instances/peers_controller.rb
@@ -2,6 +2,7 @@
 
 class Api::V1::Instances::PeersController < Api::BaseController
   before_action :require_enabled_api!
+
   skip_before_action :set_cache_headers
 
   respond_to :json
@@ -14,6 +15,6 @@ class Api::V1::Instances::PeersController < Api::BaseController
   private
 
   def require_enabled_api!
-    head 404 unless Setting.peers_api_enabled
+    head 404 unless Setting.peers_api_enabled && !whitelist_mode?
   end
 end
diff --git a/app/controllers/api/v1/instances_controller.rb b/app/controllers/api/v1/instances_controller.rb
index 93e4f0003..8d8231423 100644
--- a/app/controllers/api/v1/instances_controller.rb
+++ b/app/controllers/api/v1/instances_controller.rb
@@ -2,6 +2,7 @@
 
 class Api::V1::InstancesController < Api::BaseController
   respond_to :json
+
   skip_before_action :set_cache_headers
 
   def show
diff --git a/app/controllers/application_controller.rb b/app/controllers/application_controller.rb
index 41ce1a0ca..0d3913ee0 100644
--- a/app/controllers/application_controller.rb
+++ b/app/controllers/application_controller.rb
@@ -11,12 +11,14 @@ class ApplicationController < ActionController::Base
   include UserTrackingConcern
   include SessionTrackingConcern
   include CacheConcern
+  include DomainControlHelper
 
   helper_method :current_account
   helper_method :current_session
   helper_method :current_theme
   helper_method :single_user_mode?
   helper_method :use_seamless_external_login?
+  helper_method :whitelist_mode?
 
   rescue_from ActionController::RoutingError, with: :not_found
   rescue_from ActiveRecord::RecordNotFound, with: :not_found
@@ -38,7 +40,7 @@ class ApplicationController < ActionController::Base
   end
 
   def authorized_fetch_mode?
-    ENV['AUTHORIZED_FETCH'] == 'true'
+    ENV['AUTHORIZED_FETCH'] == 'true' || Rails.configuration.x.whitelist_mode
   end
 
   def public_fetch_mode?
diff --git a/app/controllers/concerns/account_owned_concern.rb b/app/controllers/concerns/account_owned_concern.rb
index 99c240fe9..460f71f65 100644
--- a/app/controllers/concerns/account_owned_concern.rb
+++ b/app/controllers/concerns/account_owned_concern.rb
@@ -4,6 +4,7 @@ module AccountOwnedConcern
   extend ActiveSupport::Concern
 
   included do
+    before_action :authenticate_user!, if: -> { whitelist_mode? && request.format != :json }
     before_action :set_account, if: :account_required?
     before_action :check_account_approval, if: :account_required?
     before_action :check_account_suspension, if: :account_required?
diff --git a/app/controllers/directories_controller.rb b/app/controllers/directories_controller.rb
index 594907674..d2ef76f06 100644
--- a/app/controllers/directories_controller.rb
+++ b/app/controllers/directories_controller.rb
@@ -3,7 +3,8 @@
 class DirectoriesController < ApplicationController
   layout 'public'
 
-  before_action :check_enabled
+  before_action :authenticate_user!, if: :whitelist_mode?
+  before_action :require_enabled!
   before_action :set_instance_presenter
   before_action :set_tag, only: :show
   before_action :set_tags
@@ -19,7 +20,7 @@ class DirectoriesController < ApplicationController
 
   private
 
-  def check_enabled
+  def require_enabled!
     return not_found unless Setting.profile_directory
   end
 
diff --git a/app/controllers/home_controller.rb b/app/controllers/home_controller.rb
index 42493cd78..22d507e77 100644
--- a/app/controllers/home_controller.rb
+++ b/app/controllers/home_controller.rb
@@ -55,7 +55,7 @@ class HomeController < ApplicationController
   end
 
   def default_redirect_path
-    if request.path.start_with?('/web')
+    if request.path.start_with?('/web') || whitelist_mode?
       new_user_session_path
     elsif single_user_mode?
       short_account_path(Account.local.without_suspended.where('id > 0').first)
diff --git a/app/controllers/media_controller.rb b/app/controllers/media_controller.rb
index b3b7519a1..1f693de32 100644
--- a/app/controllers/media_controller.rb
+++ b/app/controllers/media_controller.rb
@@ -5,6 +5,7 @@ class MediaController < ApplicationController
 
   skip_before_action :store_current_location
 
+  before_action :authenticate_user!, if: :whitelist_mode?
   before_action :set_media_attachment
   before_action :verify_permitted_status!
   before_action :check_playable, only: :player
diff --git a/app/controllers/media_proxy_controller.rb b/app/controllers/media_proxy_controller.rb
index 8fc18dd06..8da6c6fe0 100644
--- a/app/controllers/media_proxy_controller.rb
+++ b/app/controllers/media_proxy_controller.rb
@@ -5,6 +5,8 @@ class MediaProxyController < ApplicationController
 
   skip_before_action :store_current_location
 
+  before_action :authenticate_user!, if: :whitelist_mode?
+
   def show
     RedisLock.acquire(lock_options) do |lock|
       if lock.acquired?
diff --git a/app/controllers/public_timelines_controller.rb b/app/controllers/public_timelines_controller.rb
index 23506b990..324bdc508 100644
--- a/app/controllers/public_timelines_controller.rb
+++ b/app/controllers/public_timelines_controller.rb
@@ -3,7 +3,8 @@
 class PublicTimelinesController < ApplicationController
   layout 'public'
 
-  before_action :check_enabled
+  before_action :authenticate_user!, if: :whitelist_mode?
+  before_action :require_enabled!
   before_action :set_body_classes
   before_action :set_instance_presenter
 
@@ -16,7 +17,7 @@ class PublicTimelinesController < ApplicationController
 
   private
 
-  def check_enabled
+  def require_enabled!
     not_found unless Setting.timeline_preview
   end
 
diff --git a/app/controllers/remote_interaction_controller.rb b/app/controllers/remote_interaction_controller.rb
index cc6993c52..fa742fb0a 100644
--- a/app/controllers/remote_interaction_controller.rb
+++ b/app/controllers/remote_interaction_controller.rb
@@ -5,6 +5,7 @@ class RemoteInteractionController < ApplicationController
 
   layout 'modal'
 
+  before_action :authenticate_user!, if: :whitelist_mode?
   before_action :set_interaction_type
   before_action :set_status
   before_action :set_body_classes
diff --git a/app/controllers/tags_controller.rb b/app/controllers/tags_controller.rb
index d08e5a61a..3cd2d9e20 100644
--- a/app/controllers/tags_controller.rb
+++ b/app/controllers/tags_controller.rb
@@ -8,6 +8,7 @@ class TagsController < ApplicationController
   layout 'public'
 
   before_action :require_signature!, if: -> { request.format == :json && authorized_fetch_mode? }
+  before_action :authenticate_user!, if: :whitelist_mode?
   before_action :set_tag
   before_action :set_body_classes
   before_action :set_instance_presenter
diff --git a/app/helpers/domain_control_helper.rb b/app/helpers/domain_control_helper.rb
index efd328f81..067b2c2cd 100644
--- a/app/helpers/domain_control_helper.rb
+++ b/app/helpers/domain_control_helper.rb
@@ -12,6 +12,14 @@ module DomainControlHelper
       end
     end
 
-    DomainBlock.blocked?(domain)
+    if whitelist_mode?
+      !DomainAllow.allowed?(domain)
+    else
+      DomainBlock.blocked?(domain)
+    end
+  end
+
+  def whitelist_mode?
+    Rails.configuration.x.whitelist_mode
   end
 end
diff --git a/app/models/domain_allow.rb b/app/models/domain_allow.rb
new file mode 100644
index 000000000..85018b636
--- /dev/null
+++ b/app/models/domain_allow.rb
@@ -0,0 +1,33 @@
+# frozen_string_literal: true
+
+# == Schema Information
+#
+# Table name: domain_allows
+#
+#  id         :bigint(8)        not null, primary key
+#  domain     :string           default(""), not null
+#  created_at :datetime         not null
+#  updated_at :datetime         not null
+#
+
+class DomainAllow < ApplicationRecord
+  include DomainNormalizable
+
+  validates :domain, presence: true, uniqueness: true
+
+  scope :matches_domain, ->(value) { where(arel_table[:domain].matches("%#{value}%")) }
+
+  class << self
+    def allowed?(domain)
+      !rule_for(domain).nil?
+    end
+
+    def rule_for(domain)
+      return if domain.blank?
+
+      uri = Addressable::URI.new.tap { |u| u.host = domain.gsub(/[\/]/, '') }
+
+      find_by(domain: uri.normalized_host)
+    end
+  end
+end
diff --git a/app/models/instance.rb b/app/models/instance.rb
index 797a191e0..3c740f8a2 100644
--- a/app/models/instance.rb
+++ b/app/models/instance.rb
@@ -7,8 +7,9 @@ class Instance
 
   def initialize(resource)
     @domain         = resource.domain
-    @accounts_count = resource.is_a?(DomainBlock) ? nil : resource.accounts_count
+    @accounts_count = resource.respond_to?(:accounts_count) ? resource.accounts_count : nil
     @domain_block   = resource.is_a?(DomainBlock) ? resource : DomainBlock.rule_for(domain)
+    @domain_allow   = resource.is_a?(DomainAllow) ? resource : DomainAllow.rule_for(domain)
   end
 
   def countable?
diff --git a/app/models/instance_filter.rb b/app/models/instance_filter.rb
index 848fff53e..8bfab826d 100644
--- a/app/models/instance_filter.rb
+++ b/app/models/instance_filter.rb
@@ -12,6 +12,10 @@ class InstanceFilter
       scope = DomainBlock
       scope = scope.matches_domain(params[:by_domain]) if params[:by_domain].present?
       scope.order(id: :desc)
+    elsif params[:allowed].present?
+      scope = DomainAllow
+      scope = scope.matches_domain(params[:by_domain]) if params[:by_domain].present?
+      scope.order(id: :desc)
     else
       scope = Account.remote
       scope = scope.matches_domain(params[:by_domain]) if params[:by_domain].present?
diff --git a/app/policies/domain_allow_policy.rb b/app/policies/domain_allow_policy.rb
new file mode 100644
index 000000000..5030453bb
--- /dev/null
+++ b/app/policies/domain_allow_policy.rb
@@ -0,0 +1,11 @@
+# frozen_string_literal: true
+
+class DomainAllowPolicy < ApplicationPolicy
+  def create?
+    admin?
+  end
+
+  def destroy?
+    admin?
+  end
+end
diff --git a/app/services/concerns/payloadable.rb b/app/services/concerns/payloadable.rb
index 953740faa..7f9f21c4b 100644
--- a/app/services/concerns/payloadable.rb
+++ b/app/services/concerns/payloadable.rb
@@ -14,6 +14,6 @@ module Payloadable
   end
 
   def signing_enabled?
-    ENV['AUTHORIZED_FETCH'] != 'true'
+    ENV['AUTHORIZED_FETCH'] != 'true' && !Rails.configuration.x.whitelist_mode
   end
 end
diff --git a/app/services/unallow_domain_service.rb b/app/services/unallow_domain_service.rb
new file mode 100644
index 000000000..d4387c1a1
--- /dev/null
+++ b/app/services/unallow_domain_service.rb
@@ -0,0 +1,11 @@
+# frozen_string_literal: true
+
+class UnallowDomainService < BaseService
+  def call(domain_allow)
+    Account.where(domain: domain_allow.domain).find_each do |account|
+      SuspendAccountService.new.call(account, destroy: true)
+    end
+
+    domain_allow.destroy
+  end
+end
diff --git a/app/views/admin/domain_allows/new.html.haml b/app/views/admin/domain_allows/new.html.haml
new file mode 100644
index 000000000..52599857a
--- /dev/null
+++ b/app/views/admin/domain_allows/new.html.haml
@@ -0,0 +1,14 @@
+- content_for :header_tags do
+  = javascript_pack_tag 'admin', integrity: true, async: true, crossorigin: 'anonymous'
+
+- content_for :page_title do
+  = t('admin.domain_allows.add_new')
+
+= simple_form_for @domain_allow, url: admin_domain_allows_path do |f|
+  = render 'shared/error_messages', object: @domain_allow
+
+  .fields-group
+    = f.input :domain, wrapper: :with_label, label: t('admin.domain_blocks.domain'), required: true
+
+  .actions
+    = f.button :button, t('admin.domain_allows.add_new'), type: :submit
diff --git a/app/views/admin/instances/index.html.haml b/app/views/admin/instances/index.html.haml
index 61e578409..982dc5035 100644
--- a/app/views/admin/instances/index.html.haml
+++ b/app/views/admin/instances/index.html.haml
@@ -6,24 +6,30 @@
     %strong= t('admin.instances.moderation.title')
     %ul
       %li= filter_link_to t('admin.instances.moderation.all'), limited: nil
-      %li= filter_link_to t('admin.instances.moderation.limited'), limited: '1'
+
+      - unless whitelist_mode?
+        %li= filter_link_to t('admin.instances.moderation.limited'), limited: '1'
 
   %div{ style: 'flex: 1 1 auto; text-align: right' }
-    = link_to t('admin.domain_blocks.add_new'), new_admin_domain_block_path, class: 'button'
+    - if whitelist_mode?
+      = link_to t('admin.domain_allows.add_new'), new_admin_domain_allow_path, class: 'button'
+    - else
+      = link_to t('admin.domain_blocks.add_new'), new_admin_domain_block_path, class: 'button'
 
-= form_tag admin_instances_url, method: 'GET', class: 'simple_form' do
-  .fields-group
-    - Admin::FilterHelper::INSTANCES_FILTERS.each do |key|
-      - if params[key].present?
-        = hidden_field_tag key, params[key]
+- unless whitelist_mode?
+  = form_tag admin_instances_url, method: 'GET', class: 'simple_form' do
+    .fields-group
+      - Admin::FilterHelper::INSTANCES_FILTERS.each do |key|
+        - if params[key].present?
+          = hidden_field_tag key, params[key]
 
-    - %i(by_domain).each do |key|
-      .input.string.optional
-        = text_field_tag key, params[key], class: 'string optional', placeholder: I18n.t("admin.instances.#{key}")
+      - %i(by_domain).each do |key|
+        .input.string.optional
+          = text_field_tag key, params[key], class: 'string optional', placeholder: I18n.t("admin.instances.#{key}")
 
-    .actions
-      %button= t('admin.accounts.search')
-      = link_to t('admin.accounts.reset'), admin_instances_path, class: 'button negative'
+      .actions
+        %button= t('admin.accounts.search')
+        = link_to t('admin.accounts.reset'), admin_instances_path, class: 'button negative'
 
 %hr.spacer/
 
@@ -47,8 +53,11 @@
               - unless first_item
                 &bull;
               = t('admin.domain_blocks.rejecting_reports')
+          - elsif whitelist_mode?
+            = t('admin.accounts.whitelisted')
           - else
             = t('admin.accounts.no_limits_imposed')
       - if instance.countable?
         .trends__item__current{ title: t('admin.instances.known_accounts', count: instance.accounts_count) }= number_to_human instance.accounts_count, strip_insignificant_zeros: true
+
 = paginate paginated_instances
diff --git a/app/views/admin/instances/show.html.haml b/app/views/admin/instances/show.html.haml
index c7992a490..fbb49ba02 100644
--- a/app/views/admin/instances/show.html.haml
+++ b/app/views/admin/instances/show.html.haml
@@ -38,7 +38,9 @@
     = link_to t('admin.accounts.title'), admin_accounts_path(remote: '1', by_domain: @instance.domain), class: 'button'
 
   %div{ style: 'float: right' }
-    - if @domain_block
+    - if @domain_allow
+      = link_to t('admin.domain_allows.undo'), admin_domain_allow_path(@domain_allow), class: 'button button--destructive', data: { confirm: t('admin.accounts.are_you_sure'), method: :delete }
+    - elsif @domain_block
       = link_to t('admin.domain_blocks.undo'), admin_domain_block_path(@domain_block), class: 'button'
     - else
       = link_to t('admin.domain_blocks.add_new'), new_admin_domain_block_path(_domain: @instance.domain), class: 'button'
diff --git a/app/views/admin/settings/edit.html.haml b/app/views/admin/settings/edit.html.haml
index b3bf3849c..1e2ed3f77 100644
--- a/app/views/admin/settings/edit.html.haml
+++ b/app/views/admin/settings/edit.html.haml
@@ -42,11 +42,12 @@
 
   %hr.spacer/
 
-  .fields-group
-    = f.input :timeline_preview, as: :boolean, wrapper: :with_label, label: t('admin.settings.timeline_preview.title'), hint: t('admin.settings.timeline_preview.desc_html')
+  - unless whitelist_mode?
+    .fields-group
+      = f.input :timeline_preview, as: :boolean, wrapper: :with_label, label: t('admin.settings.timeline_preview.title'), hint: t('admin.settings.timeline_preview.desc_html')
 
-  .fields-group
-    = f.input :show_known_fediverse_at_about_page, as: :boolean, wrapper: :with_label, label: t('admin.settings.show_known_fediverse_at_about_page.title'), hint: t('admin.settings.show_known_fediverse_at_about_page.desc_html')
+    .fields-group
+      = f.input :show_known_fediverse_at_about_page, as: :boolean, wrapper: :with_label, label: t('admin.settings.show_known_fediverse_at_about_page.title'), hint: t('admin.settings.show_known_fediverse_at_about_page.desc_html')
 
   .fields-group
     = f.input :show_staff_badge, as: :boolean, wrapper: :with_label, label: t('admin.settings.show_staff_badge.title'), hint: t('admin.settings.show_staff_badge.desc_html')
@@ -54,17 +55,18 @@
   .fields-group
     = f.input :open_deletion, as: :boolean, wrapper: :with_label, label: t('admin.settings.registrations.deletion.title'), hint: t('admin.settings.registrations.deletion.desc_html')
 
-  .fields-group
-    = f.input :activity_api_enabled, as: :boolean, wrapper: :with_label, label: t('admin.settings.activity_api_enabled.title'), hint: t('admin.settings.activity_api_enabled.desc_html')
+  - unless whitelist_mode?
+    .fields-group
+      = f.input :activity_api_enabled, as: :boolean, wrapper: :with_label, label: t('admin.settings.activity_api_enabled.title'), hint: t('admin.settings.activity_api_enabled.desc_html')
 
-  .fields-group
-    = f.input :peers_api_enabled, as: :boolean, wrapper: :with_label, label: t('admin.settings.peers_api_enabled.title'), hint: t('admin.settings.peers_api_enabled.desc_html')
+    .fields-group
+      = f.input :peers_api_enabled, as: :boolean, wrapper: :with_label, label: t('admin.settings.peers_api_enabled.title'), hint: t('admin.settings.peers_api_enabled.desc_html')
 
-  .fields-group
-    = f.input :preview_sensitive_media, as: :boolean, wrapper: :with_label, label: t('admin.settings.preview_sensitive_media.title'), hint: t('admin.settings.preview_sensitive_media.desc_html')
+    .fields-group
+      = f.input :preview_sensitive_media, as: :boolean, wrapper: :with_label, label: t('admin.settings.preview_sensitive_media.title'), hint: t('admin.settings.preview_sensitive_media.desc_html')
 
-  .fields-group
-    = f.input :profile_directory, as: :boolean, wrapper: :with_label, label: t('admin.settings.profile_directory.title'), hint: t('admin.settings.profile_directory.desc_html')
+    .fields-group
+      = f.input :profile_directory, as: :boolean, wrapper: :with_label, label: t('admin.settings.profile_directory.title'), hint: t('admin.settings.profile_directory.desc_html')
 
   .fields-group
     = f.input :spam_check_enabled, as: :boolean, wrapper: :with_label, label: t('admin.settings.spam_check_enabled.title'), hint: t('admin.settings.spam_check_enabled.desc_html')
@@ -76,7 +78,7 @@
 
   .fields-group
     = f.input :closed_registrations_message, as: :text, wrapper: :with_block_label, label: t('admin.settings.registrations.closed_message.title'), hint: t('admin.settings.registrations.closed_message.desc_html'), input_html: { rows: 8 }
-    = f.input :site_extended_description, wrapper: :with_block_label, as: :text, label: t('admin.settings.site_description_extended.title'), hint: t('admin.settings.site_description_extended.desc_html'), input_html: { rows: 8 }
+    = f.input :site_extended_description, wrapper: :with_block_label, as: :text, label: t('admin.settings.site_description_extended.title'), hint: t('admin.settings.site_description_extended.desc_html'), input_html: { rows: 8 } unless whitelist_mode?
     = f.input :site_terms, wrapper: :with_block_label, as: :text, label: t('admin.settings.site_terms.title'), hint: t('admin.settings.site_terms.desc_html'), input_html: { rows: 8 }
     = f.input :custom_css, wrapper: :with_block_label, as: :text, input_html: { rows: 8 }, label: t('admin.settings.custom_css.title'), hint: t('admin.settings.custom_css.desc_html')
 
diff --git a/app/views/auth/registrations/new.html.haml b/app/views/auth/registrations/new.html.haml
index b4a7cced5..83384d737 100644
--- a/app/views/auth/registrations/new.html.haml
+++ b/app/views/auth/registrations/new.html.haml
@@ -33,7 +33,7 @@
   = f.input :invite_code, as: :hidden
 
   .fields-group
-    = f.input :agreement, as: :boolean, wrapper: :with_label, label: t('auth.checkbox_agreement_html', rules_path: about_more_path, terms_path: terms_path)
+    = f.input :agreement, as: :boolean, wrapper: :with_label, label: whitelist_mode? ? t('auth.checkbox_agreement_without_rules_html', terms_path: terms_path) : t('auth.checkbox_agreement_html', rules_path: about_more_path, terms_path: terms_path)
 
   .actions
     = f.button :button, @invite.present? ? t('auth.register') : sign_up_message, type: :submit
diff --git a/app/views/layouts/public.html.haml b/app/views/layouts/public.html.haml
index 2929ac599..69738a2f7 100644
--- a/app/views/layouts/public.html.haml
+++ b/app/views/layouts/public.html.haml
@@ -10,10 +10,13 @@
             = link_to root_url, class: 'brand' do
               = svg_logo_full
 
-            = link_to t('directories.directory'), explore_path, class: 'nav-link optional' if Setting.profile_directory
-            = link_to t('about.about_this'), about_more_path, class: 'nav-link optional'
-            = link_to t('about.apps'), 'https://joinmastodon.org/apps', class: 'nav-link optional'
+            - unless whitelist_mode?
+              = link_to t('directories.directory'), explore_path, class: 'nav-link optional' if Setting.profile_directory
+              = link_to t('about.about_this'), about_more_path, class: 'nav-link optional'
+              = link_to t('about.apps'), 'https://joinmastodon.org/apps', class: 'nav-link optional'
+
           .nav-center
+
           .nav-right
             - if user_signed_in?
               = link_to t('settings.back'), root_url, class: 'nav-link nav-button webapp-btn'
diff --git a/config/initializers/2_whitelist_mode.rb b/config/initializers/2_whitelist_mode.rb
new file mode 100644
index 000000000..a17ad07a2
--- /dev/null
+++ b/config/initializers/2_whitelist_mode.rb
@@ -0,0 +1,5 @@
+# frozen_string_literal: true
+
+Rails.application.configure do
+  config.x.whitelist_mode = ENV['WHITELIST_MODE'] == 'true'
+end
diff --git a/config/locales/en.yml b/config/locales/en.yml
index 9e1be87be..6c1a34300 100644
--- a/config/locales/en.yml
+++ b/config/locales/en.yml
@@ -186,6 +186,7 @@ en:
       username: Username
       warn: Warn
       web: Web
+      whitelisted: Whitelisted
     action_logs:
       actions:
         assigned_to_self_report: "%{name} assigned report %{target} to themselves"
@@ -269,6 +270,11 @@ en:
       week_interactions: interactions this week
       week_users_active: active this week
       week_users_new: users this week
+    domain_allows:
+      add_new: Whitelist domain
+      created_msg: Domain has been successfully whitelisted
+      destroyed_msg: Domain has been removed from the whitelist
+      undo: Remove from whitelist
     domain_blocks:
       add_new: Add new domain block
       created_msg: Domain block is now being processed
@@ -524,6 +530,7 @@ en:
     apply_for_account: Request an invite
     change_password: Password
     checkbox_agreement_html: I agree to the <a href="%{rules_path}" target="_blank">server rules</a> and <a href="%{terms_path}" target="_blank">terms of service</a>
+    checkbox_agreement_without_rules_html: I agree to the <a href="%{terms_path}" target="_blank">terms of service</a>
     delete_account: Delete account
     delete_account_html: If you wish to delete your account, you can <a href="%{path}">proceed here</a>. You will be asked for confirmation.
     didnt_get_confirmation: Didn't receive confirmation instructions?
diff --git a/config/locales/simple_form.en.yml b/config/locales/simple_form.en.yml
index 12a7ec2b3..10b30e627 100644
--- a/config/locales/simple_form.en.yml
+++ b/config/locales/simple_form.en.yml
@@ -38,6 +38,8 @@ en:
         setting_use_pending_items: Hide timeline updates behind a click instead of automatically scrolling the feed
         username: Your username will be unique on %{domain}
         whole_word: When the keyword or phrase is alphanumeric only, it will only be applied if it matches the whole word
+      domain_allow:
+        domain: This domain will be able to fetch data from this server and incoming data from it will be processed and stored
       featured_tag:
         name: 'You might want to use one of these:'
       imports:
diff --git a/config/navigation.rb b/config/navigation.rb
index 5ab2e4399..9b46da603 100644
--- a/config/navigation.rb
+++ b/config/navigation.rb
@@ -39,7 +39,7 @@ SimpleNavigation::Configuration.run do |navigation|
       s.item :accounts, safe_join([fa_icon('users fw'), t('admin.accounts.title')]), admin_accounts_url, highlights_on: %r{/admin/accounts|/admin/pending_accounts}
       s.item :invites, safe_join([fa_icon('user-plus fw'), t('admin.invites.title')]), admin_invites_path
       s.item :tags, safe_join([fa_icon('tag fw'), t('admin.tags.title')]), admin_tags_path
-      s.item :instances, safe_join([fa_icon('cloud fw'), t('admin.instances.title')]), admin_instances_url(limited: '1'), highlights_on: %r{/admin/instances|/admin/domain_blocks}, if: -> { current_user.admin? }
+      s.item :instances, safe_join([fa_icon('cloud fw'), t('admin.instances.title')]), admin_instances_url(limited: whitelist_mode? ? nil : '1'), highlights_on: %r{/admin/instances|/admin/domain_blocks|/admin/domain_allows}, if: -> { current_user.admin? }
       s.item :email_domain_blocks, safe_join([fa_icon('envelope fw'), t('admin.email_domain_blocks.title')]), admin_email_domain_blocks_url, highlights_on: %r{/admin/email_domain_blocks}, if: -> { current_user.admin? }
     end
 
diff --git a/config/routes.rb b/config/routes.rb
index b6c215888..04424bbbd 100644
--- a/config/routes.rb
+++ b/config/routes.rb
@@ -154,6 +154,7 @@ Rails.application.routes.draw do
   namespace :admin do
     get '/dashboard', to: 'dashboard#index'
 
+    resources :domain_allows, only: [:new, :create, :show, :destroy]
     resources :domain_blocks, only: [:new, :create, :show, :destroy]
     resources :email_domain_blocks, only: [:index, :new, :create, :destroy]
     resources :action_logs, only: [:index]
diff --git a/db/migrate/20190705002136_create_domain_allows.rb b/db/migrate/20190705002136_create_domain_allows.rb
new file mode 100644
index 000000000..83b0728d9
--- /dev/null
+++ b/db/migrate/20190705002136_create_domain_allows.rb
@@ -0,0 +1,9 @@
+class CreateDomainAllows < ActiveRecord::Migration[5.2]
+  def change
+    create_table :domain_allows do |t|
+      t.string :domain, default: '', null: false, index: { unique: true }
+
+      t.timestamps
+    end
+  end
+end
diff --git a/db/schema.rb b/db/schema.rb
index 1847305c7..2d83d8b76 100644
--- a/db/schema.rb
+++ b/db/schema.rb
@@ -10,7 +10,7 @@
 #
 # It's strongly recommended that you check this file into your version control system.
 
-ActiveRecord::Schema.define(version: 2019_07_26_175042) do
+ActiveRecord::Schema.define(version: 2019_07_28_084117) do
 
   # These are extensions that must be enabled in order to support this database
   enable_extension "plpgsql"
@@ -245,6 +245,13 @@ ActiveRecord::Schema.define(version: 2019_07_26_175042) do
     t.index ["account_id"], name: "index_custom_filters_on_account_id"
   end
 
+  create_table "domain_allows", force: :cascade do |t|
+    t.string "domain", default: "", null: false
+    t.datetime "created_at", null: false
+    t.datetime "updated_at", null: false
+    t.index ["domain"], name: "index_domain_allows_on_domain", unique: true
+  end
+
   create_table "domain_blocks", force: :cascade do |t|
     t.string "domain", default: "", null: false
     t.datetime "created_at", null: false
diff --git a/lib/mastodon/domains_cli.rb b/lib/mastodon/domains_cli.rb
index b081581fe..f30062363 100644
--- a/lib/mastodon/domains_cli.rb
+++ b/lib/mastodon/domains_cli.rb
@@ -12,17 +12,33 @@ module Mastodon
     end
 
     option :dry_run, type: :boolean
-    desc 'purge DOMAIN', 'Remove accounts from a DOMAIN without a trace'
+    option :whitelist_mode, type: :boolean
+    desc 'purge [DOMAIN]', 'Remove accounts from a DOMAIN without a trace'
     long_desc <<-LONG_DESC
       Remove all accounts from a given DOMAIN without leaving behind any
       records. Unlike a suspension, if the DOMAIN still exists in the wild,
       it means the accounts could return if they are resolved again.
+
+      When the --whitelist-mode option is given, instead of purging accounts
+      from a single domain, all accounts from domains that are not whitelisted
+      are removed from the database.
     LONG_DESC
-    def purge(domain)
+    def purge(domain = nil)
       removed = 0
       dry_run = options[:dry_run] ? ' (DRY RUN)' : ''
 
-      Account.where(domain: domain).find_each do |account|
+      scope = begin
+        if options[:whitelist_mode]
+          Account.remote.where.not(domain: DomainAllow.pluck(:domain))
+        elsif domain.present?
+          Account.remote.where(domain: domain)
+        else
+          say('No domain given', :red)
+          exit(1)
+        end
+      end
+
+      scope.find_each do |account|
         SuspendAccountService.new.call(account, destroy: true) unless options[:dry_run]
         removed += 1
         say('.', :green, false)
diff --git a/spec/fabricators/domain_allow_fabricator.rb b/spec/fabricators/domain_allow_fabricator.rb
new file mode 100644
index 000000000..6226b1e20
--- /dev/null
+++ b/spec/fabricators/domain_allow_fabricator.rb
@@ -0,0 +1,3 @@
+Fabricator(:domain_allow) do
+  domain "MyString"
+end
diff --git a/spec/models/domain_allow_spec.rb b/spec/models/domain_allow_spec.rb
new file mode 100644
index 000000000..e65435127
--- /dev/null
+++ b/spec/models/domain_allow_spec.rb
@@ -0,0 +1,5 @@
+require 'rails_helper'
+
+RSpec.describe DomainAllow, type: :model do
+  pending "add some examples to (or delete) #{__FILE__}"
+end
diff --git a/streaming/index.js b/streaming/index.js
index 0529804b1..304e7e046 100644
--- a/streaming/index.js
+++ b/streaming/index.js
@@ -12,6 +12,7 @@ const uuid = require('uuid');
 const fs = require('fs');
 
 const env = process.env.NODE_ENV || 'development';
+const alwaysRequireAuth = process.env.WHITELIST_MODE === 'true' || process.env.AUTHORIZED_FETCH === 'true';
 
 dotenv.config({
   path: env === 'production' ? '.env.production' : '.env',
@@ -271,7 +272,7 @@ const startWorker = (workerId) => {
 
   const wsVerifyClient = (info, cb) => {
     const location = url.parse(info.req.url, true);
-    const authRequired = !PUBLIC_STREAMS.some(stream => stream === location.query.stream);
+    const authRequired = alwaysRequireAuth || !PUBLIC_STREAMS.some(stream => stream === location.query.stream);
     const allowedScopes = [];
 
     if (authRequired) {
@@ -306,7 +307,7 @@ const startWorker = (workerId) => {
       return;
     }
 
-    const authRequired = !PUBLIC_ENDPOINTS.some(endpoint => endpoint === req.path);
+    const authRequired = alwaysRequireAuth || !PUBLIC_ENDPOINTS.some(endpoint => endpoint === req.path);
     const allowedScopes = [];
 
     if (authRequired) {
-- 
cgit 


From ff789a751a1c730e4d808410411196b76caff39c Mon Sep 17 00:00:00 2001
From: ThibG <thib@sitedethib.com>
Date: Tue, 30 Jul 2019 13:18:23 +0200
Subject: Fix boosting & unboosting preventing a boost from appearing in the TL
 (#11405)

* Fix boosting & unboosting preventing a boost from appearing in the TL

* Add tests

* Avoids side effects when aggregate_reblogs isn't true
---
 app/lib/feed_manager.rb       | 14 +++++++++-----
 spec/lib/feed_manager_spec.rb | 17 +++++++++++++++++
 2 files changed, 26 insertions(+), 5 deletions(-)

(limited to 'spec')

diff --git a/app/lib/feed_manager.rb b/app/lib/feed_manager.rb
index ed3ce6112..482b64867 100644
--- a/app/lib/feed_manager.rb
+++ b/app/lib/feed_manager.rb
@@ -35,7 +35,7 @@ class FeedManager
   end
 
   def unpush_from_home(account, status)
-    return false unless remove_from_feed(:home, account.id, status)
+    return false unless remove_from_feed(:home, account.id, status, account.user&.aggregates_reblogs?)
     redis.publish("timeline:#{account.id}", Oj.dump(event: :delete, payload: status.id.to_s))
     true
   end
@@ -53,7 +53,7 @@ class FeedManager
   end
 
   def unpush_from_list(list, status)
-    return false unless remove_from_feed(:list, list.id, status)
+    return false unless remove_from_feed(:list, list.id, status, list.account.user&.aggregates_reblogs?)
     redis.publish("timeline:list:#{list.id}", Oj.dump(event: :delete, payload: status.id.to_s))
     true
   end
@@ -105,7 +105,7 @@ class FeedManager
     oldest_home_score = redis.zrange(timeline_key, 0, 0, with_scores: true)&.first&.last&.to_i || 0
 
     from_account.statuses.select('id, reblog_of_id').where('id > ?', oldest_home_score).reorder(nil).find_each do |status|
-      remove_from_feed(:home, into_account.id, status)
+      remove_from_feed(:home, into_account.id, status, into_account.user&.aggregates_reblogs?)
     end
   end
 
@@ -275,10 +275,11 @@ class FeedManager
   # with reblogs, and returning true if a status was removed. As with
   # `add_to_feed`, this does not trigger push updates, so callers must
   # do so if appropriate.
-  def remove_from_feed(timeline_type, account_id, status)
+  def remove_from_feed(timeline_type, account_id, status, aggregate_reblogs = true)
     timeline_key = key(timeline_type, account_id)
+    reblog_key   = key(timeline_type, account_id, 'reblogs')
 
-    if status.reblog?
+    if status.reblog? && (aggregate_reblogs.nil? || aggregate_reblogs)
       # 1. If the reblogging status is not in the feed, stop.
       status_rank = redis.zrevrank(timeline_key, status.id)
       return false if status_rank.nil?
@@ -287,6 +288,7 @@ class FeedManager
       reblog_set_key = key(timeline_type, account_id, "reblogs:#{status.reblog_of_id}")
 
       redis.srem(reblog_set_key, status.id)
+      redis.zrem(reblog_key, status.reblog_of_id)
       # 3. Re-insert another reblog or original into the feed if one
       # remains in the set. We could pick a random element, but this
       # set should generally be small, and it seems ideal to show the
@@ -294,12 +296,14 @@ class FeedManager
       other_reblog = redis.smembers(reblog_set_key).map(&:to_i).min
 
       redis.zadd(timeline_key, other_reblog, other_reblog) if other_reblog
+      redis.zadd(reblog_key, other_reblog, status.reblog_of_id) if other_reblog
 
       # 4. Remove the reblogging status from the feed (as normal)
       # (outside conditional)
     else
       # If the original is getting deleted, no use for reblog references
       redis.del(key(timeline_type, account_id, "reblogs:#{status.id}"))
+      redis.srem(reblog_key, status.id)
     end
 
     redis.zrem(timeline_key, status.id)
diff --git a/spec/lib/feed_manager_spec.rb b/spec/lib/feed_manager_spec.rb
index 9bdb675e1..b996997b1 100644
--- a/spec/lib/feed_manager_spec.rb
+++ b/spec/lib/feed_manager_spec.rb
@@ -247,6 +247,23 @@ RSpec.describe FeedManager do
         expect(FeedManager.instance.push_to_home(account, reblogs.last)).to be false
       end
 
+      it 'saves a new reblog of a recently-reblogged status when previous reblog has been deleted' do
+        account = Fabricate(:account)
+        reblogged = Fabricate(:status)
+        old_reblog = Fabricate(:status, reblog: reblogged)
+
+        # The first reblog should be accepted
+        expect(FeedManager.instance.push_to_home(account, old_reblog)).to be true
+
+        # The first reblog should be successfully removed
+        expect(FeedManager.instance.unpush_from_home(account, old_reblog)).to be true
+
+        reblog = Fabricate(:status, reblog: reblogged)
+
+        # The second reblog should be accepted
+        expect(FeedManager.instance.push_to_home(account, reblog)).to be true
+      end
+
       it 'does not save a new reblog of a multiply-reblogged-then-unreblogged status' do
         account   = Fabricate(:account)
         reblogged = Fabricate(:status)
-- 
cgit 


From 92de439c04785530ce15f55cae18590136c75216 Mon Sep 17 00:00:00 2001
From: Eugen Rochko <eugen@zeonfederated.com>
Date: Tue, 30 Jul 2019 20:29:50 +0200
Subject: Change hashtag search to only return results that have trended in the
 past (#11448)

* Change hashtag search to only return results that have trended in the past

A way to eliminate typos and other one-off "junk" results

* Fix excluding exact matches that don't have a score

* Fix tests
---
 app/models/tag.rb       | 6 ++++--
 spec/models/tag_spec.rb | 4 ++--
 2 files changed, 6 insertions(+), 4 deletions(-)

(limited to 'spec')

diff --git a/app/models/tag.rb b/app/models/tag.rb
index a2d6078f4..c7f0af86d 100644
--- a/app/models/tag.rb
+++ b/app/models/tag.rb
@@ -76,9 +76,11 @@ class Tag < ApplicationRecord
     end
 
     def search_for(term, limit = 5, offset = 0)
-      pattern = sanitize_sql_like(normalize(term.strip)) + '%'
+      normalized_term = normalize(term.strip).mb_chars.downcase.to_s
+      pattern         = sanitize_sql_like(normalized_term) + '%'
 
-      Tag.where(arel_table[:name].lower.matches(pattern.mb_chars.downcase.to_s))
+      Tag.where(arel_table[:name].lower.matches(pattern))
+         .where(arel_table[:score].gt(0).or(arel_table[:name].lower.eq(normalized_term)))
          .order(Arel.sql('length(name) ASC, score DESC, name ASC'))
          .limit(limit)
          .offset(offset)
diff --git a/spec/models/tag_spec.rb b/spec/models/tag_spec.rb
index 5f07fd618..9d700849b 100644
--- a/spec/models/tag_spec.rb
+++ b/spec/models/tag_spec.rb
@@ -136,8 +136,8 @@ RSpec.describe Tag, type: :model do
     end
 
     it 'finds the exact matching tag as the first item' do
-      similar_tag = Fabricate(:tag, name: "matchlater")
-      tag = Fabricate(:tag, name: "match")
+      similar_tag = Fabricate(:tag, name: "matchlater", score: 1)
+      tag = Fabricate(:tag, name: "match", score: 1)
 
       results = Tag.search_for("match")
 
-- 
cgit 


From 8b9d0a05337b6bcf57b51abf45e21d9474bf2684 Mon Sep 17 00:00:00 2001
From: Eugen Rochko <eugen@zeonfederated.com>
Date: Thu, 1 Aug 2019 19:14:02 +0200
Subject: Remove XML version of Webfinger and remove links to Atom feeds
 (#11460)

Fix #11453
---
 app/controllers/well_known/webfinger_controller.rb | 11 +----
 app/serializers/webfinger_serializer.rb            |  1 -
 app/views/accounts/show.html.haml                  |  1 -
 app/views/well_known/webfinger/show.xml.ruby       | 51 ----------------------
 .../well_known/webfinger_controller_spec.rb        | 11 -----
 spec/requests/webfinger_request_spec.rb            | 17 --------
 6 files changed, 1 insertion(+), 91 deletions(-)
 delete mode 100644 app/views/well_known/webfinger/show.xml.ruby

(limited to 'spec')

diff --git a/app/controllers/well_known/webfinger_controller.rb b/app/controllers/well_known/webfinger_controller.rb
index 53f7f1e27..50bace217 100644
--- a/app/controllers/well_known/webfinger_controller.rb
+++ b/app/controllers/well_known/webfinger_controller.rb
@@ -9,17 +9,8 @@ module WellKnown
     def show
       @account = Account.find_local!(username_from_resource)
 
-      respond_to do |format|
-        format.any(:json, :html) do
-          render json: @account, serializer: WebfingerSerializer, content_type: 'application/jrd+json'
-        end
-
-        format.xml do
-          render content_type: 'application/xrd+xml'
-        end
-      end
-
       expires_in 3.days, public: true
+      render json: @account, serializer: WebfingerSerializer, content_type: 'application/jrd+json'
     rescue ActiveRecord::RecordNotFound
       head 404
     end
diff --git a/app/serializers/webfinger_serializer.rb b/app/serializers/webfinger_serializer.rb
index 008d0c182..c67363b8f 100644
--- a/app/serializers/webfinger_serializer.rb
+++ b/app/serializers/webfinger_serializer.rb
@@ -26,7 +26,6 @@ class WebfingerSerializer < ActiveModel::Serializer
     else
       [
         { rel: 'http://webfinger.net/rel/profile-page', type: 'text/html', href: short_account_url(object) },
-        { rel: 'http://schemas.google.com/g/2010#updates-from', type: 'application/atom+xml', href: account_url(object, format: 'atom') },
         { rel: 'self', type: 'application/activity+json', href: account_url(object) },
         { rel: 'http://ostatus.org/schema/1.0/subscribe', template: "#{authorize_interaction_url}?uri={uri}" },
       ]
diff --git a/app/views/accounts/show.html.haml b/app/views/accounts/show.html.haml
index 6846abeb6..bf7a9f5f7 100644
--- a/app/views/accounts/show.html.haml
+++ b/app/views/accounts/show.html.haml
@@ -7,7 +7,6 @@
   - if @account.user&.setting_noindex
     %meta{ name: 'robots', content: 'noindex, noarchive' }/
 
-  %link{ rel: 'alternate', type: 'application/atom+xml', href: account_url(@account, format: 'atom') }/
   %link{ rel: 'alternate', type: 'application/rss+xml', href: account_url(@account, format: 'rss') }/
   %link{ rel: 'alternate', type: 'application/activity+json', href: ActivityPub::TagManager.instance.uri_for(@account) }/
 
diff --git a/app/views/well_known/webfinger/show.xml.ruby b/app/views/well_known/webfinger/show.xml.ruby
deleted file mode 100644
index f5a54052a..000000000
--- a/app/views/well_known/webfinger/show.xml.ruby
+++ /dev/null
@@ -1,51 +0,0 @@
-doc = Ox::Document.new(version: '1.0')
-
-doc << Ox::Element.new('XRD').tap do |xrd|
-  xrd['xmlns'] = 'http://docs.oasis-open.org/ns/xri/xrd-1.0'
-
-  xrd << (Ox::Element.new('Subject') << @account.to_webfinger_s)
-
-  if @account.instance_actor?
-    xrd << (Ox::Element.new('Alias') << instance_actor_url)
-
-    xrd << Ox::Element.new('Link').tap do |link|
-      link['rel']      = 'http://webfinger.net/rel/profile-page'
-      link['type']     = 'text/html'
-      link['href']     = about_more_url(instance_actor: true)
-    end
-
-    xrd << Ox::Element.new('Link').tap do |link|
-      link['rel']      = 'self'
-      link['type']     = 'application/activity+json'
-      link['href']     = instance_actor_url
-    end
-  else
-    xrd << (Ox::Element.new('Alias') << short_account_url(@account))
-    xrd << (Ox::Element.new('Alias') << account_url(@account))
-
-    xrd << Ox::Element.new('Link').tap do |link|
-      link['rel']      = 'http://webfinger.net/rel/profile-page'
-      link['type']     = 'text/html'
-      link['href']     = short_account_url(@account)
-    end
-
-    xrd << Ox::Element.new('Link').tap do |link|
-      link['rel']      = 'http://schemas.google.com/g/2010#updates-from'
-      link['type']     = 'application/atom+xml'
-      link['href']     = account_url(@account, format: 'atom')
-    end
-
-    xrd << Ox::Element.new('Link').tap do |link|
-      link['rel']      = 'self'
-      link['type']     = 'application/activity+json'
-      link['href']     = account_url(@account)
-    end
-
-    xrd << Ox::Element.new('Link').tap do |link|
-      link['rel']      = 'http://ostatus.org/schema/1.0/subscribe'
-      link['template'] = "#{authorize_interaction_url}?acct={uri}"
-    end
-  end
-end
-
-('<?xml version="1.0" encoding="UTF-8"?>' + Ox.dump(doc, effort: :tolerant)).force_encoding('UTF-8')
diff --git a/spec/controllers/well_known/webfinger_controller_spec.rb b/spec/controllers/well_known/webfinger_controller_spec.rb
index b05745ea3..20275aa63 100644
--- a/spec/controllers/well_known/webfinger_controller_spec.rb
+++ b/spec/controllers/well_known/webfinger_controller_spec.rb
@@ -56,17 +56,6 @@ PEM
       expect(json[:aliases]).to include('https://cb6e6126.ngrok.io/@alice', 'https://cb6e6126.ngrok.io/users/alice')
     end
 
-    it 'returns JSON when account can be found' do
-      get :show, params: { resource: alice.to_webfinger_s }, format: :xml
-
-      xml = Nokogiri::XML(response.body)
-
-      expect(response).to have_http_status(200)
-      expect(response.content_type).to eq 'application/xrd+xml'
-      expect(xml.at_xpath('//xmlns:Subject').content).to eq 'acct:alice@cb6e6126.ngrok.io'
-      expect(xml.xpath('//xmlns:Alias').map(&:content)).to include('https://cb6e6126.ngrok.io/@alice', 'https://cb6e6126.ngrok.io/users/alice')
-    end
-
     it 'returns http not found when account cannot be found' do
       get :show, params: { resource: 'acct:not@existing.com' }, format: :json
 
diff --git a/spec/requests/webfinger_request_spec.rb b/spec/requests/webfinger_request_spec.rb
index 7f9e1162e..48823714e 100644
--- a/spec/requests/webfinger_request_spec.rb
+++ b/spec/requests/webfinger_request_spec.rb
@@ -12,23 +12,6 @@ describe 'The webfinger route' do
     end
   end
 
-  describe 'asking for xml format' do
-    it 'returns an xml response for xml format' do
-      get webfinger_url(resource: alice.to_webfinger_s, format: :xml)
-
-      expect(response).to have_http_status(200)
-      expect(response.content_type).to eq 'application/xrd+xml'
-    end
-
-    it 'returns an xml response for xml accept header' do
-      headers = { 'HTTP_ACCEPT' => 'application/xrd+xml' }
-      get webfinger_url(resource: alice.to_webfinger_s), headers: headers
-
-      expect(response).to have_http_status(200)
-      expect(response.content_type).to eq 'application/xrd+xml'
-    end
-  end
-
   describe 'asking for json format' do
     it 'returns a json response for json format' do
       get webfinger_url(resource: alice.to_webfinger_s, format: :json)
-- 
cgit 


From 115dab78f1cc5357281dcb593f04ac8b2629cec6 Mon Sep 17 00:00:00 2001
From: Eugen Rochko <eugen@zeonfederated.com>
Date: Mon, 5 Aug 2019 19:54:29 +0200
Subject: Change admin UI for hashtags and add back whitelisted trends (#11490)

Fix #271

Add back the `GET /api/v1/trends` API with the caveat that it does
not return tags that have not been allowed to trend by the staff.

When a hashtag begins to trend (internally) and that hashtag has
not been previously reviewed by the staff, the staff is notified.

The new admin UI for hashtags allows filtering hashtags by where
they are used (e.g. in the profile directory), whether they have
been reviewed or are pending reviewal, they show by how many people
the hashtag is used in the directory, how many people used it
today, how many statuses with it have been created today, and it
allows fixing the name of the hashtag to make it more readable.

The disallowed hashtags feature has been reworked. It is now
controlled from the admin UI for hashtags instead of from
the file `config/settings.yml`
---
 app/controllers/admin/dashboard_controller.rb      |  2 +-
 app/controllers/admin/tags_controller.rb           | 36 ++++++++-----
 app/controllers/api/v1/trends_controller.rb        | 17 ++++++
 app/controllers/settings/preferences_controller.rb |  2 +-
 app/helpers/admin/filter_helper.rb                 |  5 +-
 app/mailers/admin_mailer.rb                        | 10 ++++
 app/models/application_record.rb                   | 11 ++++
 app/models/tag.rb                                  | 60 +++++++++++++++++++---
 app/models/trending_tags.rb                        | 48 ++++++++---------
 app/models/user.rb                                 |  4 ++
 app/policies/tag_policy.rb                         |  4 +-
 app/validators/disallowed_hashtags_validator.rb    | 21 +-------
 app/views/admin/dashboard/index.html.haml          |  2 +-
 app/views/admin/tags/_tag.html.haml                | 24 +++++----
 app/views/admin/tags/index.html.haml               | 26 +++++-----
 app/views/admin/tags/show.html.haml                | 16 ++++++
 app/views/admin_mailer/new_trending_tag.text.erb   |  5 ++
 .../preferences/notifications/show.html.haml       |  1 +
 config/locales/en.yml                              | 18 ++++---
 config/locales/simple_form.en.yml                  |  7 +++
 config/navigation.rb                               |  2 +-
 config/routes.rb                                   |  9 +---
 config/settings.yml                                |  1 +
 .../20190805123746_add_capabilities_to_tags.rb     |  9 ++++
 db/schema.rb                                       |  7 ++-
 spec/controllers/admin/tags_controller_spec.rb     | 56 ++------------------
 spec/policies/tag_policy_spec.rb                   |  2 +-
 .../disallowed_hashtags_validator_spec.rb          | 26 +++++-----
 28 files changed, 258 insertions(+), 173 deletions(-)
 create mode 100644 app/controllers/api/v1/trends_controller.rb
 create mode 100644 app/views/admin/tags/show.html.haml
 create mode 100644 app/views/admin_mailer/new_trending_tag.text.erb
 create mode 100644 db/migrate/20190805123746_add_capabilities_to_tags.rb

(limited to 'spec')

diff --git a/app/controllers/admin/dashboard_controller.rb b/app/controllers/admin/dashboard_controller.rb
index e74e4755f..70afdedd7 100644
--- a/app/controllers/admin/dashboard_controller.rb
+++ b/app/controllers/admin/dashboard_controller.rb
@@ -27,7 +27,7 @@ module Admin
       @saml_enabled          = ENV['SAML_ENABLED'] == 'true'
       @pam_enabled           = ENV['PAM_ENABLED'] == 'true'
       @hidden_service        = ENV['ALLOW_ACCESS_TO_HIDDEN_SERVICE'] == 'true'
-      @trending_hashtags     = TrendingTags.get(7)
+      @trending_hashtags     = TrendingTags.get(10, filtered: false)
       @profile_directory     = Setting.profile_directory
       @timeline_preview      = Setting.timeline_preview
       @spam_check_enabled    = Setting.spam_check_enabled
diff --git a/app/controllers/admin/tags_controller.rb b/app/controllers/admin/tags_controller.rb
index e9f4f2cfa..0e9dda302 100644
--- a/app/controllers/admin/tags_controller.rb
+++ b/app/controllers/admin/tags_controller.rb
@@ -4,41 +4,49 @@ module Admin
   class TagsController < BaseController
     before_action :set_tags, only: :index
     before_action :set_tag, except: :index
-    before_action :set_filter_params
 
     def index
       authorize :tag, :index?
     end
 
-    def hide
-      authorize @tag, :hide?
-      @tag.account_tag_stat.update!(hidden: true)
-      redirect_to admin_tags_path(@filter_params)
+    def show
+      authorize @tag, :show?
     end
 
-    def unhide
-      authorize @tag, :unhide?
-      @tag.account_tag_stat.update!(hidden: false)
-      redirect_to admin_tags_path(@filter_params)
+    def update
+      authorize @tag, :update?
+
+      if @tag.update(tag_params.merge(reviewed_at: Time.now.utc))
+        redirect_to admin_tag_path(@tag.id)
+      else
+        render :show
+      end
     end
 
     private
 
     def set_tags
-      @tags = Tag.discoverable
-      @tags.merge!(Tag.hidden) if filter_params[:hidden]
+      @tags = filtered_tags.page(params[:page])
     end
 
     def set_tag
       @tag = Tag.find(params[:id])
     end
 
-    def set_filter_params
-      @filter_params = filter_params.to_hash.symbolize_keys
+    def filtered_tags
+      scope = Tag
+      scope = scope.discoverable if filter_params[:context] == 'directory'
+      scope = scope.reviewed if filter_params[:review] == 'reviewed'
+      scope = scope.pending_review if filter_params[:review] == 'pending_review'
+      scope.reorder(score: :desc)
     end
 
     def filter_params
-      params.permit(:hidden)
+      params.slice(:context, :review).permit(:context, :review)
+    end
+
+    def tag_params
+      params.require(:tag).permit(:name, :trendable, :usable, :listable)
     end
   end
 end
diff --git a/app/controllers/api/v1/trends_controller.rb b/app/controllers/api/v1/trends_controller.rb
new file mode 100644
index 000000000..bcea9857e
--- /dev/null
+++ b/app/controllers/api/v1/trends_controller.rb
@@ -0,0 +1,17 @@
+# frozen_string_literal: true
+
+class Api::V1::TrendsController < Api::BaseController
+  before_action :set_tags
+
+  respond_to :json
+
+  def index
+    render json: @tags, each_serializer: REST::TagSerializer
+  end
+
+  private
+
+  def set_tags
+    @tags = TrendingTags.get(limit_param(10))
+  end
+end
diff --git a/app/controllers/settings/preferences_controller.rb b/app/controllers/settings/preferences_controller.rb
index 742c97cdb..d548072a8 100644
--- a/app/controllers/settings/preferences_controller.rb
+++ b/app/controllers/settings/preferences_controller.rb
@@ -56,7 +56,7 @@ class Settings::PreferencesController < Settings::BaseController
       :setting_advanced_layout,
       :setting_use_blurhash,
       :setting_use_pending_items,
-      notification_emails: %i(follow follow_request reblog favourite mention digest report pending_account),
+      notification_emails: %i(follow follow_request reblog favourite mention digest report pending_account trending_tag),
       interactions: %i(must_be_follower must_be_following must_be_following_dm)
     )
   end
diff --git a/app/helpers/admin/filter_helper.rb b/app/helpers/admin/filter_helper.rb
index 0bda25974..506429e10 100644
--- a/app/helpers/admin/filter_helper.rb
+++ b/app/helpers/admin/filter_helper.rb
@@ -5,15 +5,16 @@ module Admin::FilterHelper
   REPORT_FILTERS       = %i(resolved account_id target_account_id).freeze
   INVITE_FILTER        = %i(available expired).freeze
   CUSTOM_EMOJI_FILTERS = %i(local remote by_domain shortcode).freeze
-  TAGS_FILTERS         = %i(hidden).freeze
+  TAGS_FILTERS         = %i(context review).freeze
   INSTANCES_FILTERS    = %i(limited by_domain).freeze
   FOLLOWERS_FILTERS    = %i(relationship status by_domain activity order).freeze
 
   FILTERS = ACCOUNT_FILTERS + REPORT_FILTERS + INVITE_FILTER + CUSTOM_EMOJI_FILTERS + TAGS_FILTERS + INSTANCES_FILTERS + FOLLOWERS_FILTERS
 
   def filter_link_to(text, link_to_params, link_class_params = link_to_params)
-    new_url = filtered_url_for(link_to_params)
+    new_url   = filtered_url_for(link_to_params)
     new_class = filtered_url_for(link_class_params)
+
     link_to text, new_url, class: filter_link_class(new_class)
   end
 
diff --git a/app/mailers/admin_mailer.rb b/app/mailers/admin_mailer.rb
index 9ab3e2bbd..8abce5f05 100644
--- a/app/mailers/admin_mailer.rb
+++ b/app/mailers/admin_mailer.rb
@@ -24,4 +24,14 @@ class AdminMailer < ApplicationMailer
       mail to: @me.user_email, subject: I18n.t('admin_mailer.new_pending_account.subject', instance: @instance, username: @account.username)
     end
   end
+
+  def new_trending_tag(recipient, tag)
+    @tag      = tag
+    @me       = recipient
+    @instance = Rails.configuration.x.local_domain
+
+    locale_for_account(@me) do
+      mail to: @me.user_email, subject: I18n.t('admin_mailer.new_trending_tag.subject', instance: @instance, name: @tag.name)
+    end
+  end
 end
diff --git a/app/models/application_record.rb b/app/models/application_record.rb
index 83134d41a..c1b873da6 100644
--- a/app/models/application_record.rb
+++ b/app/models/application_record.rb
@@ -2,5 +2,16 @@
 
 class ApplicationRecord < ActiveRecord::Base
   self.abstract_class = true
+
   include Remotable
+
+  def boolean_with_default(key, default_value)
+    value = attributes[key]
+
+    if value.nil?
+      default_value
+    else
+      value
+    end
+  end
 end
diff --git a/app/models/tag.rb b/app/models/tag.rb
index c7f0af86d..6a02581fa 100644
--- a/app/models/tag.rb
+++ b/app/models/tag.rb
@@ -3,11 +3,16 @@
 #
 # Table name: tags
 #
-#  id         :bigint(8)        not null, primary key
-#  name       :string           default(""), not null
-#  created_at :datetime         not null
-#  updated_at :datetime         not null
-#  score      :integer
+#  id                  :bigint(8)        not null, primary key
+#  name                :string           default(""), not null
+#  created_at          :datetime         not null
+#  updated_at          :datetime         not null
+#  score               :integer
+#  usable              :boolean
+#  trendable           :boolean
+#  listable            :boolean
+#  reviewed_at         :datetime
+#  requested_review_at :datetime
 #
 
 class Tag < ApplicationRecord
@@ -22,16 +27,17 @@ class Tag < ApplicationRecord
   HASHTAG_RE = /(?:^|[^\/\)\w])#(#{HASHTAG_NAME_RE})/i
 
   validates :name, presence: true, format: { with: /\A(#{HASHTAG_NAME_RE})\z/i }
+  validate :validate_name_change, if: -> { !new_record? && name_changed? }
 
-  scope :discoverable, -> { joins(:account_tag_stat).where(AccountTagStat.arel_table[:accounts_count].gt(0)).where(account_tag_stats: { hidden: false }).order(Arel.sql('account_tag_stats.accounts_count desc')) }
-  scope :hidden, -> { where(account_tag_stats: { hidden: true }) }
+  scope :reviewed, -> { where.not(reviewed_at: nil) }
+  scope :pending_review, -> { where(reviewed_at: nil).where.not(requested_review_at: nil) }
+  scope :discoverable, -> { where.not(listable: false).joins(:account_tag_stat).where(AccountTagStat.arel_table[:accounts_count].gt(0)).order(Arel.sql('account_tag_stats.accounts_count desc')) }
   scope :most_used, ->(account) { joins(:statuses).where(statuses: { account: account }).group(:id).order(Arel.sql('count(*) desc')) }
 
   delegate :accounts_count,
            :accounts_count=,
            :increment_count!,
            :decrement_count!,
-           :hidden?,
            to: :account_tag_stat
 
   after_save :save_account_tag_stat
@@ -48,6 +54,40 @@ class Tag < ApplicationRecord
     name
   end
 
+  def usable
+    boolean_with_default('usable', true)
+  end
+
+  alias usable? usable
+
+  def listable
+    boolean_with_default('listable', true)
+  end
+
+  alias listable? listable
+
+  def trendable
+    boolean_with_default('trendable', false)
+  end
+
+  alias trendable? trendable
+
+  def requires_review?
+    reviewed_at.nil?
+  end
+
+  def reviewed?
+    reviewed_at.present?
+  end
+
+  def requested_review?
+    requested_review_at.present?
+  end
+
+  def trending?
+    TrendingTags.trending?(self)
+  end
+
   def history
     days = []
 
@@ -117,4 +157,8 @@ class Tag < ApplicationRecord
     return unless account_tag_stat&.changed?
     account_tag_stat.save
   end
+
+  def validate_name_change
+    errors.add(:name, I18n.t('tags.does_not_match_previous_name')) unless name_was.mb_chars.casecmp(name.mb_chars).zero?
+  end
 end
diff --git a/app/models/trending_tags.rb b/app/models/trending_tags.rb
index 211c8f1dc..e9b9b25e3 100644
--- a/app/models/trending_tags.rb
+++ b/app/models/trending_tags.rb
@@ -10,20 +10,28 @@ class TrendingTags
     include Redisable
 
     def record_use!(tag, account, at_time = Time.now.utc)
-      return if disallowed_hashtags.include?(tag.name) || account.silenced? || account.bot?
+      return if account.silenced? || account.bot? || !tag.usable? || !(tag.trendable? || tag.requires_review?)
 
       increment_historical_use!(tag.id, at_time)
       increment_unique_use!(tag.id, account.id, at_time)
-      increment_vote!(tag.id, at_time)
+      increment_vote!(tag, at_time)
     end
 
-    def get(limit)
-      key     = "#{KEY}:#{Time.now.utc.beginning_of_day.to_i}"
-      tag_ids = redis.zrevrange(key, 0, limit - 1).map(&:to_i)
-      tags    = Tag.where(id: tag_ids).to_a.each_with_object({}) { |tag, h| h[tag.id] = tag }
+    def get(limit, filtered: true)
+      tag_ids = redis.zrevrange("#{KEY}:#{Time.now.utc.beginning_of_day.to_i}", 0, limit - 1).map(&:to_i)
+
+      tags = Tag.where(id: tag_ids)
+      tags = tags.where(trendable: true) if filtered
+      tags = tags.each_with_object({}) { |tag, h| h[tag.id] = tag }
+
       tag_ids.map { |tag_id| tags[tag_id] }.compact
     end
 
+    def trending?(tag)
+      rank = redis.zrevrank("#{KEY}:#{Time.now.utc.beginning_of_day.to_i}", tag.id)
+      rank.present? && rank <= 10
+    end
+
     private
 
     def increment_historical_use!(tag_id, at_time)
@@ -38,33 +46,27 @@ class TrendingTags
       redis.expire(key, EXPIRE_HISTORY_AFTER)
     end
 
-    def increment_vote!(tag_id, at_time)
+    def increment_vote!(tag, at_time)
       key      = "#{KEY}:#{at_time.beginning_of_day.to_i}"
-      expected = redis.pfcount("activity:tags:#{tag_id}:#{(at_time - 1.day).beginning_of_day.to_i}:accounts").to_f
+      expected = redis.pfcount("activity:tags:#{tag.id}:#{(at_time - 1.day).beginning_of_day.to_i}:accounts").to_f
       expected = 1.0 if expected.zero?
-      observed = redis.pfcount("activity:tags:#{tag_id}:#{at_time.beginning_of_day.to_i}:accounts").to_f
+      observed = redis.pfcount("activity:tags:#{tag.id}:#{at_time.beginning_of_day.to_i}:accounts").to_f
 
       if expected > observed || observed < THRESHOLD
-        redis.zrem(key, tag_id.to_s)
+        redis.zrem(key, tag.id)
       else
-        score = ((observed - expected)**2) / expected
-        added = redis.zadd(key, score, tag_id.to_s)
-        bump_tag_score!(tag_id) if added
+        score    = ((observed - expected)**2) / expected
+        old_rank = redis.zrevrank(key, tag.id)
+
+        redis.zadd(key, score, tag.id)
+        request_review!(tag) if (old_rank.nil? || old_rank > 10) && redis.zrevrank(key, tag.id) <= 10 && !tag.trendable? && tag.requires_review? && !tag.requested_review?
       end
 
       redis.expire(key, EXPIRE_TRENDS_AFTER)
     end
 
-    def bump_tag_score!(tag_id)
-      Tag.where(id: tag_id).update_all('score = COALESCE(score, 0) + 1')
-    end
-
-    def disallowed_hashtags
-      return @disallowed_hashtags if defined?(@disallowed_hashtags)
-
-      @disallowed_hashtags = Setting.disallowed_hashtags.nil? ? [] : Setting.disallowed_hashtags
-      @disallowed_hashtags = @disallowed_hashtags.split(' ') if @disallowed_hashtags.is_a? String
-      @disallowed_hashtags = @disallowed_hashtags.map(&:downcase)
+    def request_review!(tag)
+      User.staff.includes(:account).find_each { |u| AdminMailer.new_trending_tag(u.account, tag).deliver_later! if u.allows_trending_tag_emails? }
     end
   end
 end
diff --git a/app/models/user.rb b/app/models/user.rb
index 6806c0362..b83e26af3 100644
--- a/app/models/user.rb
+++ b/app/models/user.rb
@@ -207,6 +207,10 @@ class User < ApplicationRecord
     settings.notification_emails['pending_account']
   end
 
+  def allows_trending_tag_emails?
+    settings.notification_emails['trending_tag']
+  end
+
   def hides_network?
     @hides_network ||= settings.hide_network
   end
diff --git a/app/policies/tag_policy.rb b/app/policies/tag_policy.rb
index c63de01db..aaf70fcab 100644
--- a/app/policies/tag_policy.rb
+++ b/app/policies/tag_policy.rb
@@ -5,11 +5,11 @@ class TagPolicy < ApplicationPolicy
     staff?
   end
 
-  def hide?
+  def show?
     staff?
   end
 
-  def unhide?
+  def update?
     staff?
   end
 end
diff --git a/app/validators/disallowed_hashtags_validator.rb b/app/validators/disallowed_hashtags_validator.rb
index ee06b20f6..d745b767f 100644
--- a/app/validators/disallowed_hashtags_validator.rb
+++ b/app/validators/disallowed_hashtags_validator.rb
@@ -4,24 +4,7 @@ class DisallowedHashtagsValidator < ActiveModel::Validator
   def validate(status)
     return unless status.local? && !status.reblog?
 
-    @status = status
-    tags    = select_tags
-
-    status.errors.add(:text, I18n.t('statuses.disallowed_hashtags', tags: tags.join(', '), count: tags.size)) unless tags.empty?
-  end
-
-  private
-
-  def select_tags
-    tags = Extractor.extract_hashtags(@status.text)
-    tags.keep_if { |tag| disallowed_hashtags.include? tag.downcase }
-  end
-
-  def disallowed_hashtags
-    return @disallowed_hashtags if @disallowed_hashtags
-
-    @disallowed_hashtags = Setting.disallowed_hashtags.nil? ? [] : Setting.disallowed_hashtags
-    @disallowed_hashtags = @disallowed_hashtags.split(' ') if @disallowed_hashtags.is_a? String
-    @disallowed_hashtags = @disallowed_hashtags.map(&:downcase)
+    disallowed_hashtags = Tag.matching_name(Extractor.extract_hashtags(status.text)).reject(&:usable?)
+    status.errors.add(:text, I18n.t('statuses.disallowed_hashtags', tags: disallowed_hashtags.map(&:name).join(', '), count: disallowed_hashtags.size)) unless disallowed_hashtags.empty?
   end
 end
diff --git a/app/views/admin/dashboard/index.html.haml b/app/views/admin/dashboard/index.html.haml
index 77cc1a2a0..910896075 100644
--- a/app/views/admin/dashboard/index.html.haml
+++ b/app/views/admin/dashboard/index.html.haml
@@ -107,5 +107,5 @@
       %ul
         - @trending_hashtags.each do |tag|
           %li
-            = link_to "##{tag.name}", web_url("timelines/tag/#{tag.name}")
+            = link_to content_tag(:span, "##{tag.name}", class: !tag.trendable? && !tag.reviewed? ? 'warning-hint' : (!tag.trendable? ? 'negative-hint' : nil)), admin_tag_path(tag.id)
             %span.pull-right= number_with_delimiter(tag.history[0][:accounts].to_i)
diff --git a/app/views/admin/tags/_tag.html.haml b/app/views/admin/tags/_tag.html.haml
index 961b83f93..91af8e492 100644
--- a/app/views/admin/tags/_tag.html.haml
+++ b/app/views/admin/tags/_tag.html.haml
@@ -1,12 +1,16 @@
-%tr
-  %td
-    = link_to explore_hashtag_path(tag) do
+.directory__tag
+  = link_to admin_tag_path(tag.id) do
+    %h4
       = fa_icon 'hashtag'
       = tag.name
-  %td
-    = t('directories.people', count: tag.accounts_count)
-  %td
-    - if tag.hidden?
-      = table_link_to 'eye', t('admin.tags.unhide'), unhide_admin_tag_path(tag.id, **@filter_params), method: :post
-    - else
-      = table_link_to 'eye-slash', t('admin.tags.hide'), hide_admin_tag_path(tag.id, **@filter_params), method: :post
+
+      %small
+        = t('admin.tags.in_directory', count: tag.accounts_count)
+        &bull;
+        = t('admin.tags.unique_uses_today', count: tag.history.first[:accounts])
+
+        - if tag.trending?
+          = fa_icon 'fire fw'
+          = t('admin.tags.trending_right_now')
+
+    .trends__item__current= number_to_human tag.history.first[:uses], strip_insignificant_zeros: true
diff --git a/app/views/admin/tags/index.html.haml b/app/views/admin/tags/index.html.haml
index 4ba395860..5e4ee21f5 100644
--- a/app/views/admin/tags/index.html.haml
+++ b/app/views/admin/tags/index.html.haml
@@ -3,17 +3,19 @@
 
 .filters
   .filter-subset
-    %strong= t('admin.reports.status')
+    %strong= t('admin.tags.context')
     %ul
-      %li= filter_link_to t('admin.tags.visible'), hidden: nil
-      %li= filter_link_to t('admin.tags.hidden'), hidden: '1'
+      %li= filter_link_to t('generic.all'), context: nil
+      %li= filter_link_to t('admin.tags.directory'), context: 'directory'
 
-.table-wrapper
-  %table.table
-    %thead
-      %tr
-        %th= t('admin.tags.name')
-        %th= t('admin.tags.accounts')
-        %th
-    %tbody
-      = render @tags
+  .filter-subset
+    %strong= t('admin.tags.review')
+    %ul
+      %li= filter_link_to t('generic.all'), review: nil
+      %li= filter_link_to t('admin.tags.reviewed'), review: 'reviewed'
+      %li= filter_link_to safe_join([t('admin.accounts.moderation.pending'), "(#{Tag.pending_review.count})"], ' '), review: 'pending_review'
+
+%hr.spacer/
+
+= render @tags
+= paginate @tags
diff --git a/app/views/admin/tags/show.html.haml b/app/views/admin/tags/show.html.haml
new file mode 100644
index 000000000..27c8dc92b
--- /dev/null
+++ b/app/views/admin/tags/show.html.haml
@@ -0,0 +1,16 @@
+- content_for :page_title do
+  = "##{@tag.name}"
+
+= simple_form_for @tag, url: admin_tag_path(@tag.id) do |f|
+  = render 'shared/error_messages', object: @tag
+
+  .fields-group
+    = f.input :name, wrapper: :with_block_label
+
+  .fields-group
+    = f.input :usable, as: :boolean, wrapper: :with_label
+    = f.input :trendable, as: :boolean, wrapper: :with_label
+    = f.input :listable, as: :boolean, wrapper: :with_label
+
+  .actions
+    = f.button :button, t('generic.save_changes'), type: :submit
diff --git a/app/views/admin_mailer/new_trending_tag.text.erb b/app/views/admin_mailer/new_trending_tag.text.erb
new file mode 100644
index 000000000..f3087df37
--- /dev/null
+++ b/app/views/admin_mailer/new_trending_tag.text.erb
@@ -0,0 +1,5 @@
+<%= raw t('application_mailer.salutation', name: display_name(@me)) %>
+
+<%= raw t('admin_mailer.new_trending_tag.body', name: @tag.name) %>
+
+<%= raw t('application_mailer.view')%> <%= admin_tags_url(review: 'pending_review') %>
diff --git a/app/views/settings/preferences/notifications/show.html.haml b/app/views/settings/preferences/notifications/show.html.haml
index acc646fc3..f666ae4ff 100644
--- a/app/views/settings/preferences/notifications/show.html.haml
+++ b/app/views/settings/preferences/notifications/show.html.haml
@@ -15,6 +15,7 @@
       - if current_user.staff?
         = ff.input :report, as: :boolean, wrapper: :with_label
         = ff.input :pending_account, as: :boolean, wrapper: :with_label
+        = ff.input :trending_tag, as: :boolean, wrapper: :with_label
 
   .fields-group
     = f.simple_fields_for :notification_emails, hash_to_object(current_user.settings.notification_emails) do |ff|
diff --git a/config/locales/en.yml b/config/locales/en.yml
index 6c1a34300..9b62aac3a 100644
--- a/config/locales/en.yml
+++ b/config/locales/en.yml
@@ -483,13 +483,14 @@ en:
       title: Account statuses
       with_media: With media
     tags:
-      accounts: Accounts
-      hidden: Hidden
-      hide: Hide from directory
-      name: Hashtag
+      context: Context
+      directory: In directory
+      in_directory: "%{count} in directory"
+      review: Review status
+      reviewed: Reviewed
       title: Hashtags
-      unhide: Show in directory
-      visible: Visible
+      trending_right_now: Trending right now
+      unique_uses_today: "%{count} posting today"
     title: Administration
     warning_presets:
       add_new: Add new
@@ -505,6 +506,9 @@ en:
       body: "%{reporter} has reported %{target}"
       body_remote: Someone from %{domain} has reported %{target}
       subject: New report for %{instance} (#%{id})
+    new_trending_tag:
+      body: 'The hashtag #%{name} is trending today, but has not been previously reviewed. It will not be displayed publicly unless you allow it to, or just save the form as it is to never hear about it again.'
+      subject: New hashtag up for review on %{instance} (#%{name})
   appearance:
     advanced_web_interface: Advanced web interface
     advanced_web_interface_hint: 'If you want to make use of your entire screen width, the advanced web interface allows you to configure many different columns to see as much information at the same time as you want: Home, notifications, federated timeline, any number of lists and hashtags.'
@@ -939,6 +943,8 @@ en:
     pinned: Pinned toot
     reblogged: boosted
     sensitive_content: Sensitive content
+  tags:
+    does_not_match_previous_name: does not match the previous name
   terms:
     body_html: |
       <h2>Privacy Policy</h2>
diff --git a/config/locales/simple_form.en.yml b/config/locales/simple_form.en.yml
index 10b30e627..6fdfc9d7b 100644
--- a/config/locales/simple_form.en.yml
+++ b/config/locales/simple_form.en.yml
@@ -48,6 +48,8 @@ en:
         text: This will help us review your application
       sessions:
         otp: 'Enter the two-factor code generated by your phone app or use one of your recovery codes:'
+      tag:
+        name: You can only change the casing of the letters, for example, to make it more readable
       user:
         chosen_languages: When checked, only toots in selected languages will be displayed in public timelines
     labels:
@@ -137,6 +139,11 @@ en:
         pending_account: Send e-mail when a new account needs review
         reblog: Send e-mail when someone boosts your status
         report: Send e-mail when a new report is submitted
+        trending_tag: Send e-mail when an unreviewed hashtag is trending
+      tag:
+        listable: Allow this hashtag to appear on the profile directory
+        trendable: Allow this hashtag to appear under trends
+        usable: Allow toots to use this hashtag
     'no': 'No'
     recommended: Recommended
     required:
diff --git a/config/navigation.rb b/config/navigation.rb
index 9b46da603..38668bbf7 100644
--- a/config/navigation.rb
+++ b/config/navigation.rb
@@ -38,7 +38,7 @@ SimpleNavigation::Configuration.run do |navigation|
       s.item :reports, safe_join([fa_icon('flag fw'), t('admin.reports.title')]), admin_reports_url, highlights_on: %r{/admin/reports}
       s.item :accounts, safe_join([fa_icon('users fw'), t('admin.accounts.title')]), admin_accounts_url, highlights_on: %r{/admin/accounts|/admin/pending_accounts}
       s.item :invites, safe_join([fa_icon('user-plus fw'), t('admin.invites.title')]), admin_invites_path
-      s.item :tags, safe_join([fa_icon('tag fw'), t('admin.tags.title')]), admin_tags_path
+      s.item :tags, safe_join([fa_icon('hashtag fw'), t('admin.tags.title')]), admin_tags_path, highlights_on: %r{/admin/tags}
       s.item :instances, safe_join([fa_icon('cloud fw'), t('admin.instances.title')]), admin_instances_url(limited: whitelist_mode? ? nil : '1'), highlights_on: %r{/admin/instances|/admin/domain_blocks|/admin/domain_allows}, if: -> { current_user.admin? }
       s.item :email_domain_blocks, safe_join([fa_icon('envelope fw'), t('admin.email_domain_blocks.title')]), admin_email_domain_blocks_url, highlights_on: %r{/admin/email_domain_blocks}, if: -> { current_user.admin? }
     end
diff --git a/config/routes.rb b/config/routes.rb
index 04424bbbd..60f7d2e05 100644
--- a/config/routes.rb
+++ b/config/routes.rb
@@ -243,13 +243,7 @@ Rails.application.routes.draw do
     end
 
     resources :account_moderation_notes, only: [:create, :destroy]
-
-    resources :tags, only: [:index] do
-      member do
-        post :hide
-        post :unhide
-      end
-    end
+    resources :tags, only: [:index, :show, :update]
   end
 
   get '/admin', to: redirect('/admin/dashboard', status: 302)
@@ -311,6 +305,7 @@ Rails.application.routes.draw do
       resources :mutes,        only: [:index]
       resources :favourites,   only: [:index]
       resources :reports,      only: [:create]
+      resources :trends,       only: [:index]
       resources :filters,      only: [:index, :create, :show, :update, :destroy]
       resources :endorsements, only: [:index]
 
diff --git a/config/settings.yml b/config/settings.yml
index ad2970bb7..10180201f 100644
--- a/config/settings.yml
+++ b/config/settings.yml
@@ -43,6 +43,7 @@ defaults: &defaults
     digest: true
     report: true
     pending_account: true
+    trending_tag: true
   interactions:
     must_be_follower: false
     must_be_following: false
diff --git a/db/migrate/20190805123746_add_capabilities_to_tags.rb b/db/migrate/20190805123746_add_capabilities_to_tags.rb
new file mode 100644
index 000000000..43c7763b1
--- /dev/null
+++ b/db/migrate/20190805123746_add_capabilities_to_tags.rb
@@ -0,0 +1,9 @@
+class AddCapabilitiesToTags < ActiveRecord::Migration[5.2]
+  def change
+    add_column :tags, :usable, :boolean
+    add_column :tags, :trendable, :boolean
+    add_column :tags, :listable, :boolean
+    add_column :tags, :reviewed_at, :datetime
+    add_column :tags, :requested_review_at, :datetime
+  end
+end
diff --git a/db/schema.rb b/db/schema.rb
index e3af9c31a..d1b6825b4 100644
--- a/db/schema.rb
+++ b/db/schema.rb
@@ -10,7 +10,7 @@
 #
 # It's strongly recommended that you check this file into your version control system.
 
-ActiveRecord::Schema.define(version: 2019_07_29_185330) do
+ActiveRecord::Schema.define(version: 2019_08_05_123746) do
 
   # These are extensions that must be enabled in order to support this database
   enable_extension "plpgsql"
@@ -660,6 +660,11 @@ ActiveRecord::Schema.define(version: 2019_07_29_185330) do
     t.datetime "created_at", null: false
     t.datetime "updated_at", null: false
     t.integer "score"
+    t.boolean "usable"
+    t.boolean "trendable"
+    t.boolean "listable"
+    t.datetime "reviewed_at"
+    t.datetime "requested_review_at"
     t.index "lower((name)::text)", name: "index_tags_on_name_lower", unique: true
   end
 
diff --git a/spec/controllers/admin/tags_controller_spec.rb b/spec/controllers/admin/tags_controller_spec.rb
index 3af994071..5c1944fc7 100644
--- a/spec/controllers/admin/tags_controller_spec.rb
+++ b/spec/controllers/admin/tags_controller_spec.rb
@@ -10,62 +10,14 @@ RSpec.describe Admin::TagsController, type: :controller do
   end
 
   describe 'GET #index' do
-    before do
-      account_tag_stat = Fabricate(:tag).account_tag_stat
-      account_tag_stat.update(hidden: hidden, accounts_count: 1)
-      get :index, params: { hidden: hidden }
-    end
-
-    context 'with hidden tags' do
-      let(:hidden) { true }
-
-      it 'returns status 200' do
-        expect(response).to have_http_status(200)
-      end
-    end
-
-    context 'without hidden tags' do
-      let(:hidden) { false }
-
-      it 'returns status 200' do
-        expect(response).to have_http_status(200)
-      end
-    end
-  end
-
-  describe 'POST #hide' do
-    let(:tag) { Fabricate(:tag) }
+    let!(:tag) { Fabricate(:tag) }
 
     before do
-      tag.account_tag_stat.update(hidden: false)
-      post :hide, params: { id: tag.id }
-    end
-
-    it 'hides tag' do
-      tag.reload
-      expect(tag).to be_hidden
-    end
-
-    it 'redirects to admin_tags_path' do
-      expect(response).to redirect_to(admin_tags_path(controller.instance_variable_get(:@filter_params)))
-    end
-  end
-
-  describe 'POST #unhide' do
-    let(:tag) { Fabricate(:tag) }
-
-    before do
-      tag.account_tag_stat.update(hidden: true)
-      post :unhide, params: { id: tag.id }
-    end
-
-    it 'unhides tag' do
-      tag.reload
-      expect(tag).not_to be_hidden
+      get :index
     end
 
-    it 'redirects to admin_tags_path' do
-      expect(response).to redirect_to(admin_tags_path(controller.instance_variable_get(:@filter_params)))
+    it 'returns status 200' do
+      expect(response).to have_http_status(200)
     end
   end
 end
diff --git a/spec/policies/tag_policy_spec.rb b/spec/policies/tag_policy_spec.rb
index c7afaa7c9..c63875dc0 100644
--- a/spec/policies/tag_policy_spec.rb
+++ b/spec/policies/tag_policy_spec.rb
@@ -8,7 +8,7 @@ RSpec.describe TagPolicy do
   let(:admin)   { Fabricate(:user, admin: true).account }
   let(:john)    { Fabricate(:user).account }
 
-  permissions :index?, :hide?, :unhide? do
+  permissions :index?, :show?, :update? do
     context 'staff?' do
       it 'permits' do
         expect(subject).to permit(admin, Tag)
diff --git a/spec/validators/disallowed_hashtags_validator_spec.rb b/spec/validators/disallowed_hashtags_validator_spec.rb
index 8ec1302ab..9deec0bb9 100644
--- a/spec/validators/disallowed_hashtags_validator_spec.rb
+++ b/spec/validators/disallowed_hashtags_validator_spec.rb
@@ -3,42 +3,44 @@
 require 'rails_helper'
 
 RSpec.describe DisallowedHashtagsValidator, type: :validator do
+  let(:disallowed_tags) { [] }
+
   describe '#validate' do
     before do
-      allow_any_instance_of(described_class).to receive(:select_tags) { tags }
+      disallowed_tags.each { |name| Fabricate(:tag, name: name, usable: false) }
       described_class.new.validate(status)
     end
 
-    let(:status) { double(errors: errors, local?: local, reblog?: reblog, text: '') }
+    let(:status) { double(errors: errors, local?: local, reblog?: reblog, text: disallowed_tags.map { |x| '#' + x }.join(' ')) }
     let(:errors) { double(add: nil) }
 
-    context 'unless status.local? && !status.reblog?' do
+    context 'for a remote reblog' do
       let(:local)  { false }
       let(:reblog) { true }
 
-      it 'not calls errors.add' do
+      it 'does not add errors' do
         expect(errors).not_to have_received(:add).with(:text, any_args)
       end
     end
 
-    context 'status.local? && !status.reblog?' do
+    context 'for a local original status' do
       let(:local)  { true }
       let(:reblog) { false }
 
-      context 'tags.empty?' do
-        let(:tags) { [] }
+      context 'when does not contain any disallowed hashtags' do
+        let(:disallowed_tags) { [] }
 
-        it 'not calls errors.add' do
+        it 'does not add errors' do
           expect(errors).not_to have_received(:add).with(:text, any_args)
         end
       end
 
-      context '!tags.empty?' do
-        let(:tags) { %w(a b c) }
+      context 'when contains disallowed hashtags' do
+        let(:disallowed_tags) { %w(a b c) }
 
-        it 'calls errors.add' do
+        it 'adds an error' do
           expect(errors).to have_received(:add)
-            .with(:text, I18n.t('statuses.disallowed_hashtags', tags: tags.join(', '), count: tags.size))
+            .with(:text, I18n.t('statuses.disallowed_hashtags', tags: disallowed_tags.join(', '), count: disallowed_tags.size))
         end
       end
     end
-- 
cgit 


From 82d2069c75071138f582503c39b0996cc4cdff11 Mon Sep 17 00:00:00 2001
From: Yusuke Nakamura <unasuke@users.noreply.github.com>
Date: Tue, 6 Aug 2019 22:33:03 +0900
Subject: Bump faker from 1.9.6 to 2.1.0 and update faker api (#11489)

* Bump faker from 1.9.6 to 2.1.0

Bumps [faker](https://github.com/stympy/faker) from 1.9.6 to 2.1.0.
- [Release notes](https://github.com/stympy/faker/releases)
- [Changelog](https://github.com/stympy/faker/blob/master/CHANGELOG.md)
- [Commits](https://github.com/stympy/faker/compare/1.9.6...v2.1.0)

Signed-off-by: dependabot-preview[bot] <support@dependabot.com>

* Use faker api v2

https://github.com/stympy/faker/releases/tag/2.0
---
 Gemfile                                                |  2 +-
 Gemfile.lock                                           |  4 ++--
 .../settings/identity_proofs_controller_spec.rb        |  4 ++--
 spec/fabricators/account_fabricator.rb                 |  2 +-
 spec/fabricators/account_identity_proof_fabricator.rb  |  2 +-
 spec/models/account_spec.rb                            | 18 +++++++++---------
 spec/models/report_spec.rb                             |  2 +-
 7 files changed, 17 insertions(+), 17 deletions(-)

(limited to 'spec')

diff --git a/Gemfile b/Gemfile
index b0759d9e1..263be0ac3 100644
--- a/Gemfile
+++ b/Gemfile
@@ -112,7 +112,7 @@ end
 group :test do
   gem 'capybara', '~> 3.28'
   gem 'climate_control', '~> 0.2'
-  gem 'faker', '~> 1.9'
+  gem 'faker', '~> 2.1'
   gem 'microformats', '~> 4.1'
   gem 'rails-controller-testing', '~> 1.0'
   gem 'rspec-sidekiq', '~> 3.0'
diff --git a/Gemfile.lock b/Gemfile.lock
index a353304e2..f185f3fa5 100644
--- a/Gemfile.lock
+++ b/Gemfile.lock
@@ -229,7 +229,7 @@ GEM
       tzinfo
     excon (0.62.0)
     fabrication (2.20.2)
-    faker (1.9.6)
+    faker (2.1.0)
       i18n (>= 0.7)
     faraday (0.15.0)
       multipart-post (>= 1.2, < 3)
@@ -695,7 +695,7 @@ DEPENDENCIES
   doorkeeper (~> 5.1)
   dotenv-rails (~> 2.7)
   fabrication (~> 2.20)
-  faker (~> 1.9)
+  faker (~> 2.1)
   fast_blank (~> 1.0)
   fastimage
   fog-core (<= 2.1.0)
diff --git a/spec/controllers/settings/identity_proofs_controller_spec.rb b/spec/controllers/settings/identity_proofs_controller_spec.rb
index 2a0f91088..261e980d4 100644
--- a/spec/controllers/settings/identity_proofs_controller_spec.rb
+++ b/spec/controllers/settings/identity_proofs_controller_spec.rb
@@ -8,8 +8,8 @@ describe Settings::IdentityProofsController do
   let(:valid_token) { '1'*66 }
   let(:kbname) { 'kbuser' }
   let(:provider) { 'keybase' }
-  let(:findable_id) { Faker::Number.number(5) }
-  let(:unfindable_id) { Faker::Number.number(5) }
+  let(:findable_id) { Faker::Number.number(digits: 5) }
+  let(:unfindable_id) { Faker::Number.number(digits: 5) }
   let(:new_proof_params) do
     { provider: provider, provider_username: kbname, token: valid_token, username: user.account.username }
   end
diff --git a/spec/fabricators/account_fabricator.rb b/spec/fabricators/account_fabricator.rb
index f12464ef3..ab900c5fa 100644
--- a/spec/fabricators/account_fabricator.rb
+++ b/spec/fabricators/account_fabricator.rb
@@ -4,7 +4,7 @@ private_key = keypair.to_pem
 
 Fabricator(:account) do
   transient :suspended, :silenced
-  username            { sequence(:username) { |i| "#{Faker::Internet.user_name(nil, %w(_))}#{i}" } }
+  username            { sequence(:username) { |i| "#{Faker::Internet.user_name(separators: %w(_))}#{i}" } }
   last_webfingered_at { Time.now.utc }
   public_key          { public_key }
   private_key         { private_key }
diff --git a/spec/fabricators/account_identity_proof_fabricator.rb b/spec/fabricators/account_identity_proof_fabricator.rb
index 94f40dfd6..7b932fa96 100644
--- a/spec/fabricators/account_identity_proof_fabricator.rb
+++ b/spec/fabricators/account_identity_proof_fabricator.rb
@@ -1,7 +1,7 @@
 Fabricator(:account_identity_proof) do
   account
   provider 'keybase'
-  provider_username { sequence(:provider_username) { |i| "#{Faker::Lorem.characters(15)}" } }
+  provider_username { sequence(:provider_username) { |i| "#{Faker::Lorem.characters(number: 15)}" } }
   token { sequence(:token) { |i| "#{i}#{Faker::Crypto.sha1()*2}"[0..65] } }
   verified false
   live false
diff --git a/spec/models/account_spec.rb b/spec/models/account_spec.rb
index 3a17d540a..3eec464bd 100644
--- a/spec/models/account_spec.rb
+++ b/spec/models/account_spec.rb
@@ -607,19 +607,19 @@ RSpec.describe Account, type: :model do
       end
 
       it 'is invalid if the username is longer then 30 characters' do
-        account = Fabricate.build(:account, username: Faker::Lorem.characters(31))
+        account = Fabricate.build(:account, username: Faker::Lorem.characters(number: 31))
         account.valid?
         expect(account).to model_have_error_on_field(:username)
       end
 
       it 'is invalid if the display name is longer than 30 characters' do
-        account = Fabricate.build(:account, display_name: Faker::Lorem.characters(31))
+        account = Fabricate.build(:account, display_name: Faker::Lorem.characters(number: 31))
         account.valid?
         expect(account).to model_have_error_on_field(:display_name)
       end
 
       it 'is invalid if the note is longer than 500 characters' do
-        account = Fabricate.build(:account, note: Faker::Lorem.characters(501))
+        account = Fabricate.build(:account, note: Faker::Lorem.characters(number: 501))
         account.valid?
         expect(account).to model_have_error_on_field(:note)
       end
@@ -653,19 +653,19 @@ RSpec.describe Account, type: :model do
       end
 
       it 'is valid even if the username is longer then 30 characters' do
-        account = Fabricate.build(:account, domain: 'domain', username: Faker::Lorem.characters(31))
+        account = Fabricate.build(:account, domain: 'domain', username: Faker::Lorem.characters(number: 31))
         account.valid?
         expect(account).not_to model_have_error_on_field(:username)
       end
 
       it 'is valid even if the display name is longer than 30 characters' do
-        account = Fabricate.build(:account, domain: 'domain', display_name: Faker::Lorem.characters(31))
+        account = Fabricate.build(:account, domain: 'domain', display_name: Faker::Lorem.characters(number: 31))
         account.valid?
         expect(account).not_to model_have_error_on_field(:display_name)
       end
 
       it 'is valid even if the note is longer than 500 characters' do
-        account = Fabricate.build(:account, domain: 'domain', note: Faker::Lorem.characters(501))
+        account = Fabricate.build(:account, domain: 'domain', note: Faker::Lorem.characters(number: 501))
         account.valid?
         expect(account).not_to model_have_error_on_field(:note)
       end
@@ -804,7 +804,7 @@ RSpec.describe Account, type: :model do
   context 'when is local' do
     # Test disabled because test environment omits autogenerating keys for performance
     xit 'generates keys' do
-      account = Account.create!(domain: nil, username: Faker::Internet.user_name(nil, ['_']))
+      account = Account.create!(domain: nil, username: Faker::Internet.user_name(separators: ['_']))
       expect(account.keypair.private?).to eq true
     end
   end
@@ -812,12 +812,12 @@ RSpec.describe Account, type: :model do
   context 'when is remote' do
     it 'does not generate keys' do
       key = OpenSSL::PKey::RSA.new(1024).public_key
-      account = Account.create!(domain: 'remote', username: Faker::Internet.user_name(nil, ['_']), public_key: key.to_pem)
+      account = Account.create!(domain: 'remote', username: Faker::Internet.user_name(separators: ['_']), public_key: key.to_pem)
       expect(account.keypair.params).to eq key.params
     end
 
     it 'normalizes domain' do
-      account = Account.create!(domain: 'にゃん', username: Faker::Internet.user_name(nil, ['_']))
+      account = Account.create!(domain: 'にゃん', username: Faker::Internet.user_name(separators: ['_']))
       expect(account.domain).to eq 'xn--r9j5b5b'
     end
   end
diff --git a/spec/models/report_spec.rb b/spec/models/report_spec.rb
index a0cd0800d..312954c9d 100644
--- a/spec/models/report_spec.rb
+++ b/spec/models/report_spec.rb
@@ -125,7 +125,7 @@ describe Report do
     end
 
     it 'is invalid if comment is longer than 1000 characters' do
-      report = Fabricate.build(:report, comment: Faker::Lorem.characters(1001))
+      report = Fabricate.build(:report, comment: Faker::Lorem.characters(number: 1001))
       report.valid?
       expect(report).to model_have_error_on_field(:comment)
     end
-- 
cgit 


From bced70469a6c4aecdb3c71055f329a0f579eb14c Mon Sep 17 00:00:00 2001
From: ThibG <thib@sitedethib.com>
Date: Wed, 7 Aug 2019 20:20:23 +0200
Subject: Add domain block notes (#11515)
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

* Add database columns for adding notes to domain blocks/restrctions

* Add admin UI to set private and public comments when blocking a domain

* Add text for private and public comments on domain blocks

* Show domain block comments in admin UI

* Add comments to the domain block undo page

* Make UnblockDomainService more robust regarding upgraded domain blocks

* Allow editing domain blocks

* Rename button from “undo domain block” to “view domain block” in account admin UI

* Change test to unsilence silenced users from upgraded blocks
---
 app/controllers/admin/domain_blocks_controller.rb  | 28 ++++++++++++++++++--
 app/controllers/admin/instances_controller.rb      |  2 ++
 app/models/domain_block.rb                         | 16 +++++++-----
 app/services/block_domain_service.rb               | 11 +++++++-
 app/services/unblock_domain_service.rb             | 19 ++------------
 app/views/admin/accounts/show.html.haml            |  2 +-
 app/views/admin/domain_blocks/edit.html.haml       | 30 ++++++++++++++++++++++
 app/views/admin/domain_blocks/new.html.haml        |  6 +++++
 app/views/admin/domain_blocks/show.html.haml       | 12 +++++++++
 app/views/admin/instances/show.html.haml           | 13 ++++++++++
 app/workers/domain_block_worker.rb                 |  4 +--
 config/locales/en.yml                              |  8 ++++++
 config/routes.rb                                   |  6 ++++-
 ...20190807135426_add_comments_to_domain_blocks.rb |  7 +++++
 db/schema.rb                                       |  4 ++-
 spec/services/unblock_domain_service_spec.rb       |  2 +-
 spec/workers/domain_block_worker_spec.rb           |  2 +-
 17 files changed, 138 insertions(+), 34 deletions(-)
 create mode 100644 app/views/admin/domain_blocks/edit.html.haml
 create mode 100644 db/migrate/20190807135426_add_comments_to_domain_blocks.rb

(limited to 'spec')

diff --git a/app/controllers/admin/domain_blocks_controller.rb b/app/controllers/admin/domain_blocks_controller.rb
index 7129656da..74a36b79c 100644
--- a/app/controllers/admin/domain_blocks_controller.rb
+++ b/app/controllers/admin/domain_blocks_controller.rb
@@ -2,13 +2,17 @@
 
 module Admin
   class DomainBlocksController < BaseController
-    before_action :set_domain_block, only: [:show, :destroy]
+    before_action :set_domain_block, only: [:show, :destroy, :edit, :update]
 
     def new
       authorize :domain_block, :create?
       @domain_block = DomainBlock.new(domain: params[:_domain])
     end
 
+    def edit
+      authorize :domain_block, :create?
+    end
+
     def create
       authorize :domain_block, :create?
 
@@ -35,6 +39,22 @@ module Admin
       end
     end
 
+    def update
+      authorize :domain_block, :create?
+
+      @domain_block.update(update_params)
+
+      severity_changed = @domain_block.severity_changed?
+
+      if @domain_block.save
+        DomainBlockWorker.perform_async(@domain_block.id, severity_changed)
+        log_action :create, @domain_block
+        redirect_to admin_instances_path(limited: '1'), notice: I18n.t('admin.domain_blocks.created_msg')
+      else
+        render :edit
+      end
+    end
+
     def show
       authorize @domain_block, :show?
     end
@@ -52,8 +72,12 @@ module Admin
       @domain_block = DomainBlock.find(params[:id])
     end
 
+    def update_params
+      params.require(:domain_block).permit(:severity, :reject_media, :reject_reports, :private_comment, :public_comment)
+    end
+
     def resource_params
-      params.require(:domain_block).permit(:domain, :severity, :reject_media, :reject_reports)
+      params.require(:domain_block).permit(:domain, :severity, :reject_media, :reject_reports, :private_comment, :public_comment)
     end
   end
 end
diff --git a/app/controllers/admin/instances_controller.rb b/app/controllers/admin/instances_controller.rb
index d4f201807..b47b18f8e 100644
--- a/app/controllers/admin/instances_controller.rb
+++ b/app/controllers/admin/instances_controller.rb
@@ -21,6 +21,8 @@ module Admin
       @blocks_count    = Block.where(target_account: Account.where(domain: params[:id])).count
       @available       = DeliveryFailureTracker.available?(Account.select(:shared_inbox_url).where(domain: params[:id]).first&.shared_inbox_url)
       @media_storage   = MediaAttachment.where(account: Account.where(domain: params[:id])).sum(:file_file_size)
+      @private_comment = @domain_block&.private_comment
+      @public_comment  = @domain_block&.public_comment
     end
 
     private
diff --git a/app/models/domain_block.rb b/app/models/domain_block.rb
index 25d3b87ef..3f5b9f23e 100644
--- a/app/models/domain_block.rb
+++ b/app/models/domain_block.rb
@@ -3,13 +3,15 @@
 #
 # Table name: domain_blocks
 #
-#  id             :bigint(8)        not null, primary key
-#  domain         :string           default(""), not null
-#  created_at     :datetime         not null
-#  updated_at     :datetime         not null
-#  severity       :integer          default("silence")
-#  reject_media   :boolean          default(FALSE), not null
-#  reject_reports :boolean          default(FALSE), not null
+#  id              :bigint(8)        not null, primary key
+#  domain          :string           default(""), not null
+#  created_at      :datetime         not null
+#  updated_at      :datetime         not null
+#  severity        :integer          default("silence")
+#  reject_media    :boolean          default(FALSE), not null
+#  reject_reports  :boolean          default(FALSE), not null
+#  private_comment :text
+#  public_comment  :text
 #
 
 class DomainBlock < ApplicationRecord
diff --git a/app/services/block_domain_service.rb b/app/services/block_domain_service.rb
index c5e5e5761..0ec6be503 100644
--- a/app/services/block_domain_service.rb
+++ b/app/services/block_domain_service.rb
@@ -3,13 +3,22 @@
 class BlockDomainService < BaseService
   attr_reader :domain_block
 
-  def call(domain_block)
+  def call(domain_block, update = false)
     @domain_block = domain_block
     process_domain_block!
+    process_retroactive_updates! if update
   end
 
   private
 
+  def process_retroactive_updates!
+    # If the domain block severity has been changed, undo the appropriate limitations
+    scope = Account.by_domain_and_subdomains(domain_block.domain)
+
+    scope.where(silenced_at: domain_block.created_at).in_batches.update_all(silenced_at: nil) unless domain_block.silence?
+    scope.where(suspended_at: domain_block.created_at).in_batches.update_all(suspended_at: nil) unless domain_block.suspend?
+  end
+
   def process_domain_block!
     clear_media! if domain_block.reject_media?
 
diff --git a/app/services/unblock_domain_service.rb b/app/services/unblock_domain_service.rb
index fc262a50a..d502d9e49 100644
--- a/app/services/unblock_domain_service.rb
+++ b/app/services/unblock_domain_service.rb
@@ -10,24 +10,9 @@ class UnblockDomainService < BaseService
   end
 
   def process_retroactive_updates
-    blocked_accounts.in_batches.update_all(update_options) unless domain_block.noop?
-  end
-
-  def blocked_accounts
     scope = Account.by_domain_and_subdomains(domain_block.domain)
 
-    if domain_block.silence?
-      scope.where(silenced_at: @domain_block.created_at)
-    else
-      scope.where(suspended_at: @domain_block.created_at)
-    end
-  end
-
-  def update_options
-    { domain_block_impact => nil }
-  end
-
-  def domain_block_impact
-    domain_block.silence? ? :silenced_at : :suspended_at
+    scope.where(silenced_at: domain_block.created_at).in_batches.update_all(silenced_at: nil) unless domain_block.noop?
+    scope.where(suspended_at: domain_block.created_at).in_batches.update_all(suspended_at: nil) if domain_block.suspend?
   end
 end
diff --git a/app/views/admin/accounts/show.html.haml b/app/views/admin/accounts/show.html.haml
index 7494c9fa2..59babd3b0 100644
--- a/app/views/admin/accounts/show.html.haml
+++ b/app/views/admin/accounts/show.html.haml
@@ -174,7 +174,7 @@
 
       - unless @account.local?
         - if DomainBlock.where(domain: @account.domain).exists?
-          = link_to t('admin.domain_blocks.undo'), admin_instance_path(@account.domain), class: 'button'
+          = link_to t('admin.domain_blocks.view'), admin_instance_path(@account.domain), class: 'button'
         - else
           = link_to t('admin.domain_blocks.add_new'), new_admin_domain_block_path(_domain: @account.domain), class: 'button button--destructive'
 
diff --git a/app/views/admin/domain_blocks/edit.html.haml b/app/views/admin/domain_blocks/edit.html.haml
new file mode 100644
index 000000000..29e47ef3b
--- /dev/null
+++ b/app/views/admin/domain_blocks/edit.html.haml
@@ -0,0 +1,30 @@
+- content_for :header_tags do
+  = javascript_pack_tag 'admin', integrity: true, async: true, crossorigin: 'anonymous'
+
+- content_for :page_title do
+  = t('admin.domain_blocks.edit')
+
+= simple_form_for @domain_block, url: admin_domain_block_path(@domain_block), method: :put do |f|
+  = render 'shared/error_messages', object: @domain_block
+
+  .fields-row
+    .fields-row__column.fields-row__column-6.fields-group
+      = f.input :domain, wrapper: :with_label, label: t('admin.domain_blocks.domain'), hint: t('admin.domain_blocks.new.hint'), required: true, readonly: true, disabled: true
+
+    .fields-row__column.fields-row__column-6.fields-group
+      = f.input :severity, collection: DomainBlock.severities.keys, wrapper: :with_label, include_blank: false, label_method: lambda { |type| t("admin.domain_blocks.new.severity.#{type}") }, hint: t('admin.domain_blocks.new.severity.desc_html')
+
+  .fields-group
+    = f.input :reject_media, as: :boolean, wrapper: :with_label, label: I18n.t('admin.domain_blocks.reject_media'), hint: I18n.t('admin.domain_blocks.reject_media_hint')
+
+  .fields-group
+    = f.input :reject_reports, as: :boolean, wrapper: :with_label, label: I18n.t('admin.domain_blocks.reject_reports'), hint: I18n.t('admin.domain_blocks.reject_reports_hint')
+
+  .field-group
+    = f.input :private_comment, wrapper: :with_label, label: I18n.t('admin.domain_blocks.private_comment'), hint: t('admin.domain_blocks.private_comment_hint'), rows: 6
+
+  .field-group
+    = f.input :public_comment, wrapper: :with_label, label: I18n.t('admin.domain_blocks.public_comment'), hint: t('admin.domain_blocks.public_comment_hint'), rows: 6
+
+  .actions
+    = f.button :button, t('generic.save_changes'), type: :submit
diff --git a/app/views/admin/domain_blocks/new.html.haml b/app/views/admin/domain_blocks/new.html.haml
index 055d2fbd7..ed1581936 100644
--- a/app/views/admin/domain_blocks/new.html.haml
+++ b/app/views/admin/domain_blocks/new.html.haml
@@ -20,5 +20,11 @@
   .fields-group
     = f.input :reject_reports, as: :boolean, wrapper: :with_label, label: I18n.t('admin.domain_blocks.reject_reports'), hint: I18n.t('admin.domain_blocks.reject_reports_hint')
 
+  .field-group
+    = f.input :private_comment, wrapper: :with_label, label: I18n.t('admin.domain_blocks.private_comment'), hint: t('admin.domain_blocks.private_comment_hint'), rows: 6
+
+  .field-group
+    = f.input :public_comment, wrapper: :with_label, label: I18n.t('admin.domain_blocks.public_comment'), hint: t('admin.domain_blocks.public_comment_hint'), rows: 6
+
   .actions
     = f.button :button, t('.create'), type: :submit
diff --git a/app/views/admin/domain_blocks/show.html.haml b/app/views/admin/domain_blocks/show.html.haml
index dca4dbac7..e64aaa629 100644
--- a/app/views/admin/domain_blocks/show.html.haml
+++ b/app/views/admin/domain_blocks/show.html.haml
@@ -1,6 +1,18 @@
 - content_for :page_title do
   = t('admin.domain_blocks.show.title', domain: @domain_block.domain)
 
+- if @domain_block.private_comment.present?
+  .speech-bubble
+    .speech-bubble__bubble
+      = simple_format(h(@domain_block.private_comment))
+    .speech-bubble__owner= t 'admin.instances.private_comment'
+
+- if @domain_block.public_comment.present?
+  .speech-bubble
+    .speech-bubble__bubble
+      = simple_format(h(@domain_block.public_comment))
+    .speech-bubble__owner= t 'admin.instances.public_comment'
+
 = simple_form_for @domain_block, url: admin_domain_block_path(@domain_block), method: :delete do |f|
 
   - unless (@domain_block.noop?)
diff --git a/app/views/admin/instances/show.html.haml b/app/views/admin/instances/show.html.haml
index fbb49ba02..294c9495d 100644
--- a/app/views/admin/instances/show.html.haml
+++ b/app/views/admin/instances/show.html.haml
@@ -31,6 +31,18 @@
           = fa_icon 'times'
       .dashboard__counters__label= t 'admin.instances.delivery_available'
 
+- if @private_comment.present?
+  .speech-bubble
+    .speech-bubble__bubble
+      = simple_format(h(@private_comment))
+    .speech-bubble__owner= t 'admin.instances.private_comment'
+
+- if @public_comment.present?
+  .speech-bubble
+    .speech-bubble__bubble
+      = simple_format(h(@public_comment))
+    .speech-bubble__owner= t 'admin.instances.public_comment'
+
 %hr.spacer/
 
 %div{ style: 'overflow: hidden' }
@@ -41,6 +53,7 @@
     - if @domain_allow
       = link_to t('admin.domain_allows.undo'), admin_domain_allow_path(@domain_allow), class: 'button button--destructive', data: { confirm: t('admin.accounts.are_you_sure'), method: :delete }
     - elsif @domain_block
+      = link_to t('admin.domain_blocks.edit'), edit_admin_domain_block_path(@domain_block), class: 'button'
       = link_to t('admin.domain_blocks.undo'), admin_domain_block_path(@domain_block), class: 'button'
     - else
       = link_to t('admin.domain_blocks.add_new'), new_admin_domain_block_path(_domain: @instance.domain), class: 'button'
diff --git a/app/workers/domain_block_worker.rb b/app/workers/domain_block_worker.rb
index 884477829..35518d6b5 100644
--- a/app/workers/domain_block_worker.rb
+++ b/app/workers/domain_block_worker.rb
@@ -3,8 +3,8 @@
 class DomainBlockWorker
   include Sidekiq::Worker
 
-  def perform(domain_block_id)
-    BlockDomainService.new.call(DomainBlock.find(domain_block_id))
+  def perform(domain_block_id, update = false)
+    BlockDomainService.new.call(DomainBlock.find(domain_block_id), update)
   rescue ActiveRecord::RecordNotFound
     true
   end
diff --git a/config/locales/en.yml b/config/locales/en.yml
index 17ff24726..b677a6651 100644
--- a/config/locales/en.yml
+++ b/config/locales/en.yml
@@ -284,6 +284,7 @@ en:
       created_msg: Domain block is now being processed
       destroyed_msg: Domain block has been undone
       domain: Domain
+      edit: Edit domain block
       existing_domain_block_html: You have already imposed stricter limits on %{name}, you need to <a href="%{unblock_url}">unblock it</a> first.
       new:
         create: Create block
@@ -294,6 +295,10 @@ en:
           silence: Silence
           suspend: Suspend
         title: New domain block
+      private_comment: Private comment
+      private_comment_hint: Comment about this domain limitation for internal use by the moderators.
+      public_comment: Public comment
+      public_comment_hint: Comment about this domain limitation for the general public, if advertising the list of domain limitations is enabled.
       reject_media: Reject media files
       reject_media_hint: Removes locally stored media files and refuses to download any in the future. Irrelevant for suspensions
       reject_reports: Reject reports
@@ -313,6 +318,7 @@ en:
         title: Undo domain block for %{domain}
         undo: Undo
       undo: Undo domain block
+      view: View domain block
     email_domain_blocks:
       add_new: Add new
       created_msg: Successfully added e-mail domain to blacklist
@@ -336,6 +342,8 @@ en:
         all: All
         limited: Limited
         title: Moderation
+      private_comment: Private comment
+      public_comment: Public comment
       title: Federation
       total_blocked_by_us: Blocked by us
       total_followed_by_them: Followed by them
diff --git a/config/routes.rb b/config/routes.rb
index 60f7d2e05..9c33b8190 100644
--- a/config/routes.rb
+++ b/config/routes.rb
@@ -155,7 +155,11 @@ Rails.application.routes.draw do
     get '/dashboard', to: 'dashboard#index'
 
     resources :domain_allows, only: [:new, :create, :show, :destroy]
-    resources :domain_blocks, only: [:new, :create, :show, :destroy]
+    resources :domain_blocks, only: [:new, :create, :show, :destroy, :update] do
+      member do
+        get :edit
+      end
+    end
     resources :email_domain_blocks, only: [:index, :new, :create, :destroy]
     resources :action_logs, only: [:index]
     resources :warning_presets, except: [:new]
diff --git a/db/migrate/20190807135426_add_comments_to_domain_blocks.rb b/db/migrate/20190807135426_add_comments_to_domain_blocks.rb
new file mode 100644
index 000000000..b660a71ad
--- /dev/null
+++ b/db/migrate/20190807135426_add_comments_to_domain_blocks.rb
@@ -0,0 +1,7 @@
+class AddCommentsToDomainBlocks < ActiveRecord::Migration[5.2]
+  def change
+    add_column :domain_blocks, :private_comment, :text
+    add_column :domain_blocks, :public_comment, :text
+  end
+end
+
diff --git a/db/schema.rb b/db/schema.rb
index d1b6825b4..f8fc6a821 100644
--- a/db/schema.rb
+++ b/db/schema.rb
@@ -10,7 +10,7 @@
 #
 # It's strongly recommended that you check this file into your version control system.
 
-ActiveRecord::Schema.define(version: 2019_08_05_123746) do
+ActiveRecord::Schema.define(version: 2019_08_07_135426) do
 
   # These are extensions that must be enabled in order to support this database
   enable_extension "plpgsql"
@@ -259,6 +259,8 @@ ActiveRecord::Schema.define(version: 2019_08_05_123746) do
     t.integer "severity", default: 0
     t.boolean "reject_media", default: false, null: false
     t.boolean "reject_reports", default: false, null: false
+    t.text "private_comment"
+    t.text "public_comment"
     t.index ["domain"], name: "index_domain_blocks_on_domain", unique: true
   end
 
diff --git a/spec/services/unblock_domain_service_spec.rb b/spec/services/unblock_domain_service_spec.rb
index 619aefb5c..27dbc92ad 100644
--- a/spec/services/unblock_domain_service_spec.rb
+++ b/spec/services/unblock_domain_service_spec.rb
@@ -31,7 +31,7 @@ describe UnblockDomainService, type: :service do
       subject.call(@domain_block)
       expect_deleted_domain_block
       expect(@suspended.reload.suspended?).to be false
-      expect(@silenced.reload.silenced?).to be true
+      expect(@silenced.reload.silenced?).to be false
       expect(@independently_suspended.reload.suspended?).to be true
       expect(@independently_silenced.reload.silenced?).to be true
     end
diff --git a/spec/workers/domain_block_worker_spec.rb b/spec/workers/domain_block_worker_spec.rb
index c4138501f..48b3e38c4 100644
--- a/spec/workers/domain_block_worker_spec.rb
+++ b/spec/workers/domain_block_worker_spec.rb
@@ -14,7 +14,7 @@ describe DomainBlockWorker do
       result = subject.perform(domain_block.id)
 
       expect(result).to be_nil
-      expect(service).to have_received(:call).with(domain_block)
+      expect(service).to have_received(:call).with(domain_block, false)
     end
 
     it 'calls domain block service for relevant domain block' do
-- 
cgit 


From 8fdff2748ffbc4ce3d36c75f2bd33182b641e895 Mon Sep 17 00:00:00 2001
From: Eugen Rochko <eugen@zeonfederated.com>
Date: Fri, 16 Aug 2019 01:24:03 +0200
Subject: Add more accurate account search (#11537)

* Add more accurate account search

When ElasticSearch is available, a more accurate search is implemented:

- Using edge n-gram index for acct and display name
- Using asciifolding and cjk width normalization on display names
- Using Gaussian decay on account activity for additional scoring (recency)
- Using followers/friends ratio for additional scoring (spamminess)
- Using followers number for additional scoring (size)

The exact match precedence only takes effect when the input conforms
to the username format and the username part of it is complete, i.e.
when the user started typing the domain part.

* Support single-letter usernames

* Fix tests

* Fix not picking up account updates

* Add weights and normalization for scores, skip zero terms queries

* Use local counts for accounts index, adjust search parameters

* Fix mistakes

* Using updated_at of accounts is inadequate for remote accounts
---
 app/chewy/accounts_index.rb                  |  36 +++++
 app/models/account.rb                        |   6 +
 app/models/account_stat.rb                   |   2 +
 app/services/account_search_service.rb       | 189 ++++++++++++++++++---------
 spec/services/account_search_service_spec.rb | 122 ++++-------------
 5 files changed, 194 insertions(+), 161 deletions(-)
 create mode 100644 app/chewy/accounts_index.rb

(limited to 'spec')

diff --git a/app/chewy/accounts_index.rb b/app/chewy/accounts_index.rb
new file mode 100644
index 000000000..e11b80039
--- /dev/null
+++ b/app/chewy/accounts_index.rb
@@ -0,0 +1,36 @@
+# frozen_string_literal: true
+
+class AccountsIndex < Chewy::Index
+  settings index: { refresh_interval: '5m' }, analysis: {
+    analyzer: {
+      content: {
+        tokenizer: 'whitespace',
+        filter: %w(lowercase asciifolding cjk_width),
+      },
+
+      edge_ngram: {
+        tokenizer: 'edge_ngram',
+        filter: %w(lowercase asciifolding cjk_width),
+      },
+    },
+
+    tokenizer: {
+      edge_ngram: {
+        type: 'edge_ngram',
+        min_gram: 1,
+        max_gram: 15,
+      },
+    },
+  }
+
+  define_type ::Account.searchable.includes(:account_stat), delete_if: ->(account) { account.destroyed? || !account.searchable? } do
+    root date_detection: false do
+      field :id, type: 'long'
+      field :display_name, type: 'text', analyzer: 'edge_ngram', search_analyzer: 'content'
+      field :acct, type: 'text', analyzer: 'edge_ngram', search_analyzer: 'content', value: ->(account) { [account.username, account.domain].compact.join('@') }
+      field :following_count, type: 'long', value: ->(account) { account.active_relationships.count }
+      field :followers_count, type: 'long', value: ->(account) { account.passive_relationships.count }
+      field :last_status_at, type: 'date', value: ->(account) { account.last_status_at || account.created_at }
+    end
+  end
+end
diff --git a/app/models/account.rb b/app/models/account.rb
index 60c06aaf0..392cc625f 100644
--- a/app/models/account.rb
+++ b/app/models/account.rb
@@ -127,6 +127,8 @@ class Account < ApplicationRecord
 
   delegate :chosen_languages, to: :user, prefix: false, allow_nil: true
 
+  update_index('accounts#account', :self) if Chewy.enabled?
+
   def local?
     domain.nil?
   end
@@ -169,6 +171,10 @@ class Account < ApplicationRecord
     subscription_expires_at.present?
   end
 
+  def searchable?
+    !(suspended? || moved?)
+  end
+
   def possibly_stale?
     last_webfingered_at.nil? || last_webfingered_at <= 1.day.ago
   end
diff --git a/app/models/account_stat.rb b/app/models/account_stat.rb
index 9813aa84f..6d1097cec 100644
--- a/app/models/account_stat.rb
+++ b/app/models/account_stat.rb
@@ -16,6 +16,8 @@
 class AccountStat < ApplicationRecord
   belongs_to :account, inverse_of: :account_stat
 
+  update_index('accounts#account', :account) if Chewy.enabled?
+
   def increment_count!(key)
     update(attributes_for_increment(key))
   end
diff --git a/app/services/account_search_service.rb b/app/services/account_search_service.rb
index e1874d045..2d602a31d 100644
--- a/app/services/account_search_service.rb
+++ b/app/services/account_search_service.rb
@@ -4,105 +4,134 @@ class AccountSearchService < BaseService
   attr_reader :query, :limit, :offset, :options, :account
 
   def call(query, account = nil, options = {})
-    @query   = query.strip
-    @limit   = options[:limit].to_i
-    @offset  = options[:offset].to_i
-    @options = options
-    @account = account
+    @acct_hint = query.start_with?('@')
+    @query     = query.strip.gsub(/\A@/, '')
+    @limit     = options[:limit].to_i
+    @offset    = options[:offset].to_i
+    @options   = options
+    @account   = account
 
-    search_service_results
+    search_service_results.compact.uniq
   end
 
   private
 
   def search_service_results
-    return [] if query_blank_or_hashtag? || limit < 1
+    return [] if query.blank? || limit < 1
 
-    if resolving_non_matching_remote_account?
-      [ResolveAccountService.new.call("#{query_username}@#{query_domain}")].compact
-    else
-      search_results_and_exact_match.compact.uniq
-    end
+    [exact_match] + search_results
   end
 
-  def resolving_non_matching_remote_account?
-    offset.zero? && options[:resolve] && !exact_match? && !domain_is_local?
-  end
+  def exact_match
+    return unless offset.zero? && username_complete?
 
-  def search_results_and_exact_match
-    return search_results.to_a unless offset.zero?
+    return @exact_match if defined?(@exact_match)
 
-    results = [exact_match]
+    @exact_match = begin
+      if options[:resolve]
+        ResolveAccountService.new.call(query)
+      elsif domain_is_local?
+        Account.find_local(query_username)
+      else
+        Account.find_remote(query_username, query_domain)
+      end
+    end
+  end
 
-    return results if exact_match? && limit == 1
+  def search_results
+    return [] if limit_for_non_exact_results.zero?
 
-    results + search_results.to_a
+    @search_results ||= begin
+      if Chewy.enabled?
+        from_elasticsearch
+      else
+        from_database
+      end
+    end
   end
 
-  def query_blank_or_hashtag?
-    query.blank? || query.start_with?('#')
+  def from_database
+    if account
+      advanced_search_results
+    else
+      simple_search_results
+    end
   end
 
-  def split_query_string
-    @split_query_string ||= query.gsub(/\A@/, '').split('@')
+  def advanced_search_results
+    Account.advanced_search_for(terms_for_query, account, limit_for_non_exact_results, options[:following], offset)
   end
 
-  def query_username
-    @query_username ||= split_query_string.first || ''
+  def simple_search_results
+    Account.search_for(terms_for_query, limit_for_non_exact_results, offset)
   end
 
-  def query_domain
-    @query_domain ||= query_without_split? ? nil : split_query_string.last
-  end
+  def from_elasticsearch
+    must_clauses   = [{ multi_match: { query: terms_for_query, fields: likely_acct? ? %w(acct) : %w(acct^2 display_name), type: 'best_fields' } }]
+    should_clauses = []
 
-  def query_without_split?
-    split_query_string.size == 1
-  end
+    if account
+      return [] if options[:following] && following_ids.empty?
 
-  def domain_is_local?
-    @domain_is_local ||= TagManager.instance.local_domain?(query_domain)
-  end
+      if options[:following]
+        must_clauses << { terms: { id: following_ids } }
+      elsif following_ids.any?
+        should_clauses << { terms: { id: following_ids, boost: 100 } }
+      end
+    end
 
-  def search_from
-    options[:following] && account ? account.following : Account
-  end
+    query     = { bool: { must: must_clauses, should: should_clauses } }
+    functions = [reputation_score_function, followers_score_function, time_distance_function]
 
-  def exact_match?
-    exact_match.present?
-  end
+    records = AccountsIndex.query(function_score: { query: query, functions: functions, boost_mode: 'multiply', score_mode: 'avg' })
+                           .limit(limit_for_non_exact_results)
+                           .offset(offset)
+                           .objects
+                           .compact
 
-  def exact_match
-    return @exact_match if defined?(@exact_match)
+    ActiveRecord::Associations::Preloader.new.preload(records, :account_stat)
 
-    @exact_match = begin
-      if domain_is_local?
-        search_from.without_suspended.find_local(query_username)
-      else
-        search_from.without_suspended.find_remote(query_username, query_domain)
-      end
-    end
+    records
   end
 
-  def search_results
-    @search_results ||= begin
-      if account
-        advanced_search_results
-      else
-        simple_search_results
-      end
-    end
+  def reputation_score_function
+    {
+      script_score: {
+        script: {
+          source: "(doc['followers_count'].value + 0.0) / (doc['followers_count'].value + doc['following_count'].value + 1)",
+        },
+      },
+    }
   end
 
-  def advanced_search_results
-    Account.advanced_search_for(terms_for_query, account, limit_for_non_exact_results, options[:following], offset)
+  def followers_score_function
+    {
+      field_value_factor: {
+        field: 'followers_count',
+        modifier: 'log2p',
+        missing: 1,
+      },
+    }
   end
 
-  def simple_search_results
-    Account.search_for(terms_for_query, limit_for_non_exact_results, offset)
+  def time_distance_function
+    {
+      gauss: {
+        last_status_at: {
+          scale: '30d',
+          offset: '30d',
+          decay: 0.3,
+        },
+      },
+    }
+  end
+
+  def following_ids
+    @following_ids ||= account.active_relationships.pluck(:target_account_id)
   end
 
   def limit_for_non_exact_results
-    if offset.zero? && exact_match?
+    if exact_match?
       limit - 1
     else
       limit
@@ -113,7 +142,39 @@ class AccountSearchService < BaseService
     if domain_is_local?
       query_username
     else
-      "#{query_username} #{query_domain}"
+      query
     end
   end
+
+  def split_query_string
+    @split_query_string ||= query.split('@')
+  end
+
+  def query_username
+    @query_username ||= split_query_string.first || ''
+  end
+
+  def query_domain
+    @query_domain ||= query_without_split? ? nil : split_query_string.last
+  end
+
+  def query_without_split?
+    split_query_string.size == 1
+  end
+
+  def domain_is_local?
+    @domain_is_local ||= TagManager.instance.local_domain?(query_domain)
+  end
+
+  def exact_match?
+    exact_match.present?
+  end
+
+  def username_complete?
+    query.include?('@') && "@#{query}" =~ Account::MENTION_RE
+  end
+
+  def likely_acct?
+    @acct_hint || username_complete?
+  end
 end
diff --git a/spec/services/account_search_service_spec.rb b/spec/services/account_search_service_spec.rb
index 7b071b378..5b7182586 100644
--- a/spec/services/account_search_service_spec.rb
+++ b/spec/services/account_search_service_spec.rb
@@ -1,126 +1,56 @@
 require 'rails_helper'
 
 describe AccountSearchService, type: :service do
-  describe '.call' do
-    describe 'with a query to ignore' do
+  describe '#call' do
+    context 'with a query to ignore' do
       it 'returns empty array for missing query' do
         results = subject.call('', nil, limit: 10)
 
         expect(results).to eq []
       end
-      it 'returns empty array for hashtag query' do
-        results = subject.call('#tag', nil, limit: 10)
 
-        expect(results).to eq []
-      end
       it 'returns empty array for limit zero' do
         Fabricate(:account, username: 'match')
+
         results = subject.call('match', nil, limit: 0)
 
         expect(results).to eq []
       end
     end
 
-    describe 'searching for a simple term that is not an exact match' do
+    context 'searching for a simple term that is not an exact match' do
       it 'does not return a nil entry in the array for the exact match' do
-        match = Fabricate(:account, username: 'matchingusername')
-
+        account = Fabricate(:account, username: 'matchingusername')
         results = subject.call('match', nil, limit: 5)
-        expect(results).to eq [match]
-      end
-    end
 
-    describe 'searching local and remote users' do
-      describe "when only '@'" do
-        before do
-          allow(Account).to receive(:find_local)
-          allow(Account).to receive(:search_for)
-          subject.call('@', nil, limit: 10)
-        end
-
-        it 'uses find_local with empty query to look for local accounts' do
-          expect(Account).to have_received(:find_local).with('')
-        end
-      end
-
-      describe 'when no domain' do
-        before do
-          allow(Account).to receive(:find_local)
-          allow(Account).to receive(:search_for)
-          subject.call('one', nil, limit: 10)
-        end
-
-        it 'uses find_local to look for local accounts' do
-          expect(Account).to have_received(:find_local).with('one')
-        end
-
-        it 'uses search_for to find matches' do
-          expect(Account).to have_received(:search_for).with('one', 10, 0)
-        end
-      end
-
-      describe 'when there is a domain' do
-        before do
-          allow(Account).to receive(:find_remote)
-        end
-
-        it 'uses find_remote to look for remote accounts' do
-          subject.call('two@example.com', nil, limit: 10)
-          expect(Account).to have_received(:find_remote).with('two', 'example.com')
-        end
-
-        describe 'and there is no account provided' do
-          it 'uses search_for to find matches' do
-            allow(Account).to receive(:search_for)
-            subject.call('two@example.com', nil, limit: 10, resolve: false)
-
-            expect(Account).to have_received(:search_for).with('two example.com', 10, 0)
-          end
-        end
-
-        describe 'and there is an account provided' do
-          it 'uses advanced_search_for to find matches' do
-            account = Fabricate(:account)
-            allow(Account).to receive(:advanced_search_for)
-            subject.call('two@example.com', account, limit: 10, resolve: false)
-
-            expect(Account).to have_received(:advanced_search_for).with('two example.com', account, 10, nil, 0)
-          end
-        end
+        expect(results).to eq [account]
       end
     end
 
-    describe 'with an exact match' do
-      it 'returns exact match first, and does not return duplicates' do
-        partial = Fabricate(:account, username: 'exactness')
-        exact = Fabricate(:account, username: 'exact')
-
-        results = subject.call('exact', nil, limit: 10)
-        expect(results.size).to eq 2
-        expect(results).to eq [exact, partial]
-      end
-    end
-
-    describe 'when there is a local domain' do
+    context 'when there is a local domain' do
       around do |example|
         before = Rails.configuration.x.local_domain
+
         example.run
+
         Rails.configuration.x.local_domain = before
       end
 
       it 'returns exact match first' do
         remote     = Fabricate(:account, username: 'a', domain: 'remote', display_name: 'e')
         remote_too = Fabricate(:account, username: 'b', domain: 'remote', display_name: 'e')
-        exact = Fabricate(:account, username: 'e')
+        exact      = Fabricate(:account, username: 'e')
+
         Rails.configuration.x.local_domain = 'example.com'
 
         results = subject.call('e@example.com', nil, limit: 2)
+
         expect(results.size).to eq 2
         expect(results).to eq([exact, remote]).or eq([exact, remote_too])
       end
     end
 
-    describe 'when there is a domain but no exact match' do
+    context 'when there is a domain but no exact match' do
       it 'follows the remote account when resolve is true' do
         service = double(call: nil)
         allow(ResolveAccountService).to receive(:new).and_return(service)
@@ -138,23 +68,21 @@ describe AccountSearchService, type: :service do
       end
     end
 
-    describe 'should not include suspended accounts' do
-      it 'returns the fuzzy match first, and does not return suspended exacts' do
-        partial = Fabricate(:account, username: 'exactness')
-        exact = Fabricate(:account, username: 'exact', suspended: true)
+    it 'returns the fuzzy match first, and does not return suspended exacts' do
+      partial = Fabricate(:account, username: 'exactness')
+      exact   = Fabricate(:account, username: 'exact', suspended: true)
+      results = subject.call('exact', nil, limit: 10)
 
-        results = subject.call('exact', nil, limit: 10)
-        expect(results.size).to eq 1
-        expect(results).to eq [partial]
-      end
+      expect(results.size).to eq 1
+      expect(results).to eq [partial]
+    end
 
-      it "does not return suspended remote accounts" do
-        remote = Fabricate(:account, username: 'a', domain: 'remote', display_name: 'e', suspended: true)
+    it "does not return suspended remote accounts" do
+      remote  = Fabricate(:account, username: 'a', domain: 'remote', display_name: 'e', suspended: true)
+      results = subject.call('a@example.com', nil, limit: 2)
 
-        results = subject.call('a@example.com', nil, limit: 2)
-        expect(results.size).to eq 0
-        expect(results).to eq []
-      end
+      expect(results.size).to eq 0
+      expect(results).to eq []
     end
   end
 end
-- 
cgit 


From e5cee8062f15191d9dd388a65f6caa104abfd559 Mon Sep 17 00:00:00 2001
From: Eugen Rochko <eugen@zeonfederated.com>
Date: Fri, 16 Aug 2019 19:15:05 +0200
Subject: Fix blurhash and autoplay not working on public pages (#11585)

---
 app/controllers/home_controller.rb             | 16 ----------------
 app/controllers/public_timelines_controller.rb |  7 +------
 app/controllers/shares_controller.rb           | 18 +-----------------
 app/controllers/tags_controller.rb             |  5 -----
 app/helpers/application_helper.rb              | 21 +++++++++++++++++++++
 app/serializers/initial_state_serializer.rb    |  5 +++++
 app/views/home/index.html.haml                 |  3 +--
 app/views/layouts/public.html.haml             |  1 +
 app/views/public_timelines/show.html.haml      |  1 -
 app/views/shares/show.html.haml                |  2 +-
 app/views/tags/show.html.haml                  |  1 -
 spec/controllers/home_controller_spec.rb       | 10 ----------
 spec/controllers/shares_controller_spec.rb     |  5 +----
 13 files changed, 32 insertions(+), 63 deletions(-)

(limited to 'spec')

diff --git a/app/controllers/home_controller.rb b/app/controllers/home_controller.rb
index 22d507e77..7c8a18d17 100644
--- a/app/controllers/home_controller.rb
+++ b/app/controllers/home_controller.rb
@@ -3,7 +3,6 @@
 class HomeController < ApplicationController
   before_action :authenticate_user!
   before_action :set_referrer_policy_header
-  before_action :set_initial_state_json
 
   def index
     @body_classes = 'app-body'
@@ -39,21 +38,6 @@ class HomeController < ApplicationController
     redirect_to(matches ? tag_path(CGI.unescape(matches[:tag])) : default_redirect_path)
   end
 
-  def set_initial_state_json
-    serializable_resource = ActiveModelSerializers::SerializableResource.new(InitialStatePresenter.new(initial_state_params), serializer: InitialStateSerializer)
-    @initial_state_json   = serializable_resource.to_json
-  end
-
-  def initial_state_params
-    {
-      settings: Web::Setting.find_by(user: current_user)&.data || {},
-      push_subscription: current_account.user.web_push_subscription(current_session),
-      current_account: current_account,
-      token: current_session.token,
-      admin: Account.find_local(Setting.site_contact_username.strip.gsub(/\A@/, '')),
-    }
-  end
-
   def default_redirect_path
     if request.path.start_with?('/web') || whitelist_mode?
       new_user_session_path
diff --git a/app/controllers/public_timelines_controller.rb b/app/controllers/public_timelines_controller.rb
index 324bdc508..1332ba16c 100644
--- a/app/controllers/public_timelines_controller.rb
+++ b/app/controllers/public_timelines_controller.rb
@@ -8,12 +8,7 @@ class PublicTimelinesController < ApplicationController
   before_action :set_body_classes
   before_action :set_instance_presenter
 
-  def show
-    @initial_state_json = ActiveModelSerializers::SerializableResource.new(
-      InitialStatePresenter.new(settings: { known_fediverse: Setting.show_known_fediverse_at_about_page }, token: current_session&.token),
-      serializer: InitialStateSerializer
-    ).to_json
-  end
+  def show; end
 
   private
 
diff --git a/app/controllers/shares_controller.rb b/app/controllers/shares_controller.rb
index af605b98f..6546b8497 100644
--- a/app/controllers/shares_controller.rb
+++ b/app/controllers/shares_controller.rb
@@ -6,26 +6,10 @@ class SharesController < ApplicationController
   before_action :authenticate_user!
   before_action :set_body_classes
 
-  def show
-    serializable_resource = ActiveModelSerializers::SerializableResource.new(InitialStatePresenter.new(initial_state_params), serializer: InitialStateSerializer)
-    @initial_state_json   = serializable_resource.to_json
-  end
+  def show; end
 
   private
 
-  def initial_state_params
-    text = [params[:title], params[:text], params[:url]].compact.join(' ')
-
-    {
-      settings: Web::Setting.find_by(user: current_user)&.data || {},
-      push_subscription: current_account.user.web_push_subscription(current_session),
-      current_account: current_account,
-      token: current_session.token,
-      admin: Account.find_local(Setting.site_contact_username.strip.gsub(/\A@/, '')),
-      text: text,
-    }
-  end
-
   def set_body_classes
     @body_classes = 'modal-layout compose-standalone'
   end
diff --git a/app/controllers/tags_controller.rb b/app/controllers/tags_controller.rb
index 5a6fcc8fd..4dfa05264 100644
--- a/app/controllers/tags_controller.rb
+++ b/app/controllers/tags_controller.rb
@@ -17,11 +17,6 @@ class TagsController < ApplicationController
     respond_to do |format|
       format.html do
         expires_in 0, public: true
-
-        @initial_state_json = ActiveModelSerializers::SerializableResource.new(
-          InitialStatePresenter.new(settings: {}, token: current_session&.token),
-          serializer: InitialStateSerializer
-        ).to_json
       end
 
       format.rss do
diff --git a/app/helpers/application_helper.rb b/app/helpers/application_helper.rb
index 9d113263d..23cbb1d93 100644
--- a/app/helpers/application_helper.rb
+++ b/app/helpers/application_helper.rb
@@ -122,4 +122,25 @@ module ApplicationHelper
     text = word_wrap(text, line_width: line_width - 2, break_sequence: break_sequence)
     text.split("\n").map { |line| '> ' + line }.join("\n")
   end
+
+  def render_initial_state
+    state_params = {
+      settings: {
+        known_fediverse: Setting.show_known_fediverse_at_about_page,
+      },
+
+      text: [params[:title], params[:text], params[:url]].compact.join(' '),
+    }
+
+    if user_signed_in?
+      state_params[:settings]          = state_params[:settings].merge(Web::Setting.find_by(user: current_user)&.data || {})
+      state_params[:push_subscription] = current_account.user.web_push_subscription(current_session)
+      state_params[:current_account]   = current_account
+      state_params[:token]             = current_session.token
+      state_params[:admin]             = Account.find_local(Setting.site_contact_username.strip.gsub(/\A@/, ''))
+    end
+
+    json = ActiveModelSerializers::SerializableResource.new(InitialStatePresenter.new(state_params), serializer: InitialStateSerializer).to_json
+    content_tag(:script, json_escape(json).html_safe, id: 'initial-state', type: 'application/json')
+  end
 end
diff --git a/app/serializers/initial_state_serializer.rb b/app/serializers/initial_state_serializer.rb
index 2cebef2c0..fb53ea314 100644
--- a/app/serializers/initial_state_serializer.rb
+++ b/app/serializers/initial_state_serializer.rb
@@ -38,6 +38,11 @@ class InitialStateSerializer < ActiveModel::Serializer
       store[:use_pending_items] = object.current_account.user.setting_use_pending_items
       store[:is_staff]          = object.current_account.user.staff?
       store[:trends]            = Setting.trends && object.current_account.user.setting_trends
+    else
+      store[:auto_play_gif] = Setting.auto_play_gif
+      store[:display_media] = Setting.display_media
+      store[:reduce_motion] = Setting.reduce_motion
+      store[:use_blurhash]  = Setting.use_blurhash
     end
 
     store
diff --git a/app/views/home/index.html.haml b/app/views/home/index.html.haml
index 4c7fac0b6..30c7aab19 100644
--- a/app/views/home/index.html.haml
+++ b/app/views/home/index.html.haml
@@ -5,8 +5,7 @@
   = preload_link_tag asset_pack_path('features/notifications.js'), crossorigin: 'anonymous'
 
   %meta{name: 'applicationServerKey', content: Rails.configuration.x.vapid_public_key}
-  %script#initial-state{ type: 'application/json' }!= json_escape(@initial_state_json)
-
+  = render_initial_state
   = javascript_pack_tag 'application', integrity: true, crossorigin: 'anonymous'
 
 .app-holder#mastodon{ data: { props: Oj.dump(default_props) } }
diff --git a/app/views/layouts/public.html.haml b/app/views/layouts/public.html.haml
index 69738a2f7..b9179e23d 100644
--- a/app/views/layouts/public.html.haml
+++ b/app/views/layouts/public.html.haml
@@ -1,4 +1,5 @@
 - content_for :header_tags do
+  = render_initial_state
   = javascript_pack_tag 'public', integrity: true, crossorigin: 'anonymous'
 
 - content_for :content do
diff --git a/app/views/public_timelines/show.html.haml b/app/views/public_timelines/show.html.haml
index 913d5d855..07215efdf 100644
--- a/app/views/public_timelines/show.html.haml
+++ b/app/views/public_timelines/show.html.haml
@@ -3,7 +3,6 @@
 
 - content_for :header_tags do
   %meta{ name: 'robots', content: 'noindex' }/
-  %script#initial-state{ type: 'application/json' }!= json_escape(@initial_state_json)
   = javascript_pack_tag 'about', integrity: true, crossorigin: 'anonymous'
 
 .page-header
diff --git a/app/views/shares/show.html.haml b/app/views/shares/show.html.haml
index 44b6f145f..f2f5479a7 100644
--- a/app/views/shares/show.html.haml
+++ b/app/views/shares/show.html.haml
@@ -1,5 +1,5 @@
 - content_for :header_tags do
-  %script#initial-state{ type: 'application/json' }!= json_escape(@initial_state_json)
+  = render_initial_state
   = javascript_pack_tag 'share', integrity: true, crossorigin: 'anonymous'
 
 #mastodon-compose{ data: { props: Oj.dump(default_props) } }
diff --git a/app/views/tags/show.html.haml b/app/views/tags/show.html.haml
index cf4246822..630702277 100644
--- a/app/views/tags/show.html.haml
+++ b/app/views/tags/show.html.haml
@@ -5,7 +5,6 @@
   %meta{ name: 'robots', content: 'noindex' }/
   %link{ rel: 'alternate', type: 'application/rss+xml', href: tag_url(@tag, format: 'rss') }/
 
-  %script#initial-state{ type: 'application/json' }!= json_escape(@initial_state_json)
   = javascript_pack_tag 'about', integrity: true, crossorigin: 'anonymous'
   = render 'og'
 
diff --git a/spec/controllers/home_controller_spec.rb b/spec/controllers/home_controller_spec.rb
index f43cf0c27..941f1dd91 100644
--- a/spec/controllers/home_controller_spec.rb
+++ b/spec/controllers/home_controller_spec.rb
@@ -27,16 +27,6 @@ RSpec.describe HomeController, type: :controller do
         subject
         expect(assigns(:body_classes)).to eq 'app-body'
       end
-
-      it 'assigns @initial_state_json' do
-        subject
-        initial_state_json = json_str_to_hash(assigns(:initial_state_json))
-        expect(initial_state_json[:meta]).to_not be_nil
-        expect(initial_state_json[:compose]).to_not be_nil
-        expect(initial_state_json[:accounts]).to_not be_nil
-        expect(initial_state_json[:settings]).to_not be_nil
-        expect(initial_state_json[:media_attachments]).to_not be_nil
-      end
     end
   end
 end
diff --git a/spec/controllers/shares_controller_spec.rb b/spec/controllers/shares_controller_spec.rb
index a74e9af56..d6de3016a 100644
--- a/spec/controllers/shares_controller_spec.rb
+++ b/spec/controllers/shares_controller_spec.rb
@@ -7,15 +7,12 @@ describe SharesController do
   before { sign_in user }
 
   describe 'GTE #show' do
-    subject(:initial_state_json) { JSON.parse(assigns(:initial_state_json), symbolize_names: true) }
     subject(:body_classes) { assigns(:body_classes) }
 
     before { get :show, params: { title: 'test title', text: 'test text', url: 'url1 url2' } }
 
-    it 'assigns json' do
+    it 'returns http success' do
       expect(response).to have_http_status :ok
-      expect(initial_state_json[:compose][:text]).to eq 'test title test text url1 url2'
-      expect(initial_state_json[:meta][:me]).to eq user.account.id.to_s
       expect(body_classes).to eq 'modal-layout compose-standalone'
     end
   end
-- 
cgit 


From cc0a55cf9aead00e1cb044649f84c2187e0e4a35 Mon Sep 17 00:00:00 2001
From: Eugen Rochko <eugen@zeonfederated.com>
Date: Sun, 18 Aug 2019 03:45:51 +0200
Subject: Add more accurate hashtag search (#11579)

* Add more accurate hashtag search

Using ElasticSearch to index hashtags with edge n-grams and score
them by usage within the last 7 days since last activity. Only
hashtags that have been reviewed and are listable can appear in
searches, unless they match the query exactly

* Fix search analyzer dropping non-ascii characters
---
 app/chewy/tags_index.rb                            | 37 ++++++++++
 app/models/tag.rb                                  | 14 ++--
 app/models/trending_tags.rb                        |  3 +
 app/services/account_search_service.rb             |  2 +-
 app/services/search_service.rb                     |  8 +--
 app/services/tag_search_service.rb                 | 82 ++++++++++++++++++++++
 config/locales/simple_form.en.yml                  |  2 +-
 .../20190815225426_add_last_status_at_to_tags.rb   |  6 ++
 db/schema.rb                                       |  4 +-
 spec/models/tag_spec.rb                            |  4 +-
 10 files changed, 149 insertions(+), 13 deletions(-)
 create mode 100644 app/chewy/tags_index.rb
 create mode 100644 app/services/tag_search_service.rb
 create mode 100644 db/migrate/20190815225426_add_last_status_at_to_tags.rb

(limited to 'spec')

diff --git a/app/chewy/tags_index.rb b/app/chewy/tags_index.rb
new file mode 100644
index 000000000..300fc128f
--- /dev/null
+++ b/app/chewy/tags_index.rb
@@ -0,0 +1,37 @@
+# frozen_string_literal: true
+
+class TagsIndex < Chewy::Index
+  settings index: { refresh_interval: '15m' }, analysis: {
+    analyzer: {
+      content: {
+        tokenizer: 'keyword',
+        filter: %w(lowercase asciifolding cjk_width),
+      },
+
+      edge_ngram: {
+        tokenizer: 'edge_ngram',
+        filter: %w(lowercase asciifolding cjk_width),
+      },
+    },
+
+    tokenizer: {
+      edge_ngram: {
+        type: 'edge_ngram',
+        min_gram: 2,
+        max_gram: 15,
+      },
+    },
+  }
+
+  define_type ::Tag.listable, delete_if: ->(tag) { tag.destroyed? || !tag.listable? } do
+    root date_detection: false do
+      field :name, type: 'text', analyzer: 'content' do
+        field :edge_ngram, type: 'text', analyzer: 'edge_ngram', search_analyzer: 'content'
+      end
+
+      field :reviewed, type: 'boolean', value: ->(tag) { tag.reviewed? }
+      field :usage, type: 'long', value: ->(tag) { tag.history.reduce(0) { |total, day| total + day[:accounts].to_i } }
+      field :last_status_at, type: 'date', value: ->(tag) { tag.last_status_at || tag.created_at }
+    end
+  end
+end
diff --git a/app/models/tag.rb b/app/models/tag.rb
index 1364d1dba..5094d973d 100644
--- a/app/models/tag.rb
+++ b/app/models/tag.rb
@@ -13,6 +13,8 @@
 #  listable            :boolean
 #  reviewed_at         :datetime
 #  requested_review_at :datetime
+#  last_status_at      :datetime
+#  last_trend_at       :datetime
 #
 
 class Tag < ApplicationRecord
@@ -33,7 +35,8 @@ class Tag < ApplicationRecord
   scope :unreviewed, -> { where(reviewed_at: nil) }
   scope :pending_review, -> { unreviewed.where.not(requested_review_at: nil) }
   scope :usable, -> { where(usable: [true, nil]) }
-  scope :discoverable, -> { where(listable: [true, nil]).joins(:account_tag_stat).where(AccountTagStat.arel_table[:accounts_count].gt(0)).order(Arel.sql('account_tag_stats.accounts_count desc')) }
+  scope :listable, -> { where(listable: [true, nil]) }
+  scope :discoverable, -> { listable.joins(:account_tag_stat).where(AccountTagStat.arel_table[:accounts_count].gt(0)).order(Arel.sql('account_tag_stats.accounts_count desc')) }
   scope :most_used, ->(account) { joins(:statuses).where(statuses: { account: account }).group(:id).order(Arel.sql('count(*) desc')) }
 
   delegate :accounts_count,
@@ -44,6 +47,8 @@ class Tag < ApplicationRecord
 
   after_save :save_account_tag_stat
 
+  update_index('tags#tag', :self) if Chewy.enabled?
+
   def account_tag_stat
     super || build_account_tag_stat
   end
@@ -121,9 +126,10 @@ class Tag < ApplicationRecord
       normalized_term = normalize(term.strip).mb_chars.downcase.to_s
       pattern         = sanitize_sql_like(normalized_term) + '%'
 
-      Tag.where(arel_table[:name].lower.matches(pattern))
-         .where(arel_table[:score].gt(0).or(arel_table[:name].lower.eq(normalized_term)))
-         .order(Arel.sql('length(name) ASC, score DESC, name ASC'))
+      Tag.listable
+         .where(arel_table[:name].lower.matches(pattern))
+         .where(arel_table[:name].lower.eq(normalized_term).or(arel_table[:reviewed_at].not_eq(nil)))
+         .order(Arel.sql('length(name) ASC, name ASC'))
          .limit(limit)
          .offset(offset)
     end
diff --git a/app/models/trending_tags.rb b/app/models/trending_tags.rb
index 3d60a7fea..e4ce988c1 100644
--- a/app/models/trending_tags.rb
+++ b/app/models/trending_tags.rb
@@ -17,6 +17,9 @@ class TrendingTags
       increment_historical_use!(tag.id, at_time)
       increment_unique_use!(tag.id, account.id, at_time)
       increment_vote!(tag, at_time)
+
+      tag.update(last_status_at: Time.now.utc) if tag.last_status_at.nil? || tag.last_status_at < 12.hours.ago
+      tag.update(last_trend_at: Time.now.utc)  if trending?(tag) && (tag.last_trend_at.nil? || tag.last_trend_at < 12.hours.ago)
     end
 
     def get(limit, filtered: true)
diff --git a/app/services/account_search_service.rb b/app/services/account_search_service.rb
index d7bccdfe0..01caaefa9 100644
--- a/app/services/account_search_service.rb
+++ b/app/services/account_search_service.rb
@@ -109,7 +109,7 @@ class AccountSearchService < BaseService
       field_value_factor: {
         field: 'followers_count',
         modifier: 'log2p',
-        missing: 1,
+        missing: 0,
       },
     }
   end
diff --git a/app/services/search_service.rb b/app/services/search_service.rb
index 786d34b15..fe601bbf4 100644
--- a/app/services/search_service.rb
+++ b/app/services/search_service.rb
@@ -57,10 +57,10 @@ class SearchService < BaseService
   end
 
   def perform_hashtags_search!
-    Tag.search_for(
-      @query.gsub(/\A#/, ''),
-      @limit,
-      @offset
+    TagSearchService.new.call(
+      @query,
+      limit: @limit,
+      offset: @offset
     )
   end
 
diff --git a/app/services/tag_search_service.rb b/app/services/tag_search_service.rb
new file mode 100644
index 000000000..64dd76bb7
--- /dev/null
+++ b/app/services/tag_search_service.rb
@@ -0,0 +1,82 @@
+# frozen_string_literal: true
+
+class TagSearchService < BaseService
+  def call(query, options = {})
+    @query  = query.strip.gsub(/\A#/, '')
+    @offset = options[:offset].to_i
+    @limit  = options[:limit].to_i
+
+    if Chewy.enabled?
+      from_elasticsearch
+    else
+      from_database
+    end
+  end
+
+  private
+
+  def from_elasticsearch
+    query = {
+      function_score: {
+        query: {
+          multi_match: {
+            query: @query,
+            fields: %w(name.edge_ngram name),
+            type: 'most_fields',
+            operator: 'and',
+          },
+        },
+
+        functions: [
+          {
+            field_value_factor: {
+              field: 'usage',
+              modifier: 'log2p',
+              missing: 0,
+            },
+          },
+
+          {
+            gauss: {
+              last_status_at: {
+                scale: '7d',
+                offset: '14d',
+                decay: 0.5,
+              },
+            },
+          },
+        ],
+
+        boost_mode: 'multiply',
+      },
+    }
+
+    filter = {
+      bool: {
+        should: [
+          {
+            term: {
+              reviewed: {
+                value: true,
+              },
+            },
+          },
+
+          {
+            term: {
+              name: {
+                value: @query,
+              },
+            },
+          },
+        ],
+      },
+    }
+
+    TagsIndex.query(query).filter(filter).limit(@limit).offset(@offset).objects.compact
+  end
+
+  def from_database
+    Tag.search_for(@query, @limit, @offset)
+  end
+end
diff --git a/config/locales/simple_form.en.yml b/config/locales/simple_form.en.yml
index e15d5904f..98f0843d0 100644
--- a/config/locales/simple_form.en.yml
+++ b/config/locales/simple_form.en.yml
@@ -142,7 +142,7 @@ en:
         report: Send e-mail when a new report is submitted
         trending_tag: Send e-mail when an unreviewed hashtag is trending
       tag:
-        listable: Allow this hashtag to appear on the profile directory
+        listable: Allow this hashtag to appear in searches and on the profile directory
         trendable: Allow this hashtag to appear under trends
         usable: Allow toots to use this hashtag
     'no': 'No'
diff --git a/db/migrate/20190815225426_add_last_status_at_to_tags.rb b/db/migrate/20190815225426_add_last_status_at_to_tags.rb
new file mode 100644
index 000000000..d83537c47
--- /dev/null
+++ b/db/migrate/20190815225426_add_last_status_at_to_tags.rb
@@ -0,0 +1,6 @@
+class AddLastStatusAtToTags < ActiveRecord::Migration[5.2]
+  def change
+    add_column :tags, :last_status_at, :datetime
+    add_column :tags, :last_trend_at, :datetime
+  end
+end
diff --git a/db/schema.rb b/db/schema.rb
index f8fc6a821..0da20d62f 100644
--- a/db/schema.rb
+++ b/db/schema.rb
@@ -10,7 +10,7 @@
 #
 # It's strongly recommended that you check this file into your version control system.
 
-ActiveRecord::Schema.define(version: 2019_08_07_135426) do
+ActiveRecord::Schema.define(version: 2019_08_15_225426) do
 
   # These are extensions that must be enabled in order to support this database
   enable_extension "plpgsql"
@@ -667,6 +667,8 @@ ActiveRecord::Schema.define(version: 2019_08_07_135426) do
     t.boolean "listable"
     t.datetime "reviewed_at"
     t.datetime "requested_review_at"
+    t.datetime "last_status_at"
+    t.datetime "last_trend_at"
     t.index "lower((name)::text)", name: "index_tags_on_name_lower", unique: true
   end
 
diff --git a/spec/models/tag_spec.rb b/spec/models/tag_spec.rb
index 9d700849b..2bb30fb57 100644
--- a/spec/models/tag_spec.rb
+++ b/spec/models/tag_spec.rb
@@ -136,8 +136,8 @@ RSpec.describe Tag, type: :model do
     end
 
     it 'finds the exact matching tag as the first item' do
-      similar_tag = Fabricate(:tag, name: "matchlater", score: 1)
-      tag = Fabricate(:tag, name: "match", score: 1)
+      similar_tag = Fabricate(:tag, name: "matchlater", reviewed_at: Time.now.utc)
+      tag = Fabricate(:tag, name: "match", reviewed_at: Time.now.utc)
 
       results = Tag.search_for("match")
 
-- 
cgit 


From 97192d9a77c0b4b68afe50d6a94d87110a8adbcd Mon Sep 17 00:00:00 2001
From: Eugen Rochko <eugen@zeonfederated.com>
Date: Thu, 22 Aug 2019 04:17:12 +0200
Subject: Fix remote and staff-removed statuses leaving media behind for a day
 (#11638)

The reason for unattaching media instead of removing it is to support
delete & redraft functionality, but remote or staff-removed statuses
will never be redrafted, so the media should be deleted immediately
---
 app/controllers/api/v1/statuses_controller.rb               |  2 +-
 app/lib/activitypub/activity/delete.rb                      |  2 +-
 app/models/form/status_batch.rb                             |  2 +-
 app/services/batched_remove_status_service.rb               |  2 +-
 app/services/remove_status_service.rb                       | 12 ++++++++++++
 app/workers/removal_worker.rb                               |  4 ++--
 spec/controllers/admin/reported_statuses_controller_spec.rb |  2 +-
 spec/controllers/admin/statuses_controller_spec.rb          |  2 +-
 spec/models/form/status_batch_spec.rb                       |  4 ++--
 9 files changed, 22 insertions(+), 10 deletions(-)

(limited to 'spec')

diff --git a/app/controllers/api/v1/statuses_controller.rb b/app/controllers/api/v1/statuses_controller.rb
index 71a505c26..39ca56482 100644
--- a/app/controllers/api/v1/statuses_controller.rb
+++ b/app/controllers/api/v1/statuses_controller.rb
@@ -53,7 +53,7 @@ class Api::V1::StatusesController < Api::BaseController
     @status = Status.where(account_id: current_user.account).find(params[:id])
     authorize @status, :destroy?
 
-    RemovalWorker.perform_async(@status.id)
+    RemovalWorker.perform_async(@status.id, redraft: true)
 
     render json: @status, serializer: REST::StatusSerializer, source_requested: true
   end
diff --git a/app/lib/activitypub/activity/delete.rb b/app/lib/activitypub/activity/delete.rb
index 1f2b40c15..345060462 100644
--- a/app/lib/activitypub/activity/delete.rb
+++ b/app/lib/activitypub/activity/delete.rb
@@ -70,7 +70,7 @@ class ActivityPub::Activity::Delete < ActivityPub::Activity
   end
 
   def delete_now!
-    RemoveStatusService.new.call(@status)
+    RemoveStatusService.new.call(@status, redraft: false)
   end
 
   def payload
diff --git a/app/models/form/status_batch.rb b/app/models/form/status_batch.rb
index 933dfdaca..831d8b7c5 100644
--- a/app/models/form/status_batch.rb
+++ b/app/models/form/status_batch.rb
@@ -34,7 +34,7 @@ class Form::StatusBatch
 
   def delete_statuses
     Status.where(id: status_ids).reorder(nil).find_each do |status|
-      RemovalWorker.perform_async(status.id)
+      RemovalWorker.perform_async(status.id, redraft: false)
       Tombstone.find_or_create_by(uri: status.uri, account: status.account, by_moderator: true)
       log_action :destroy, status
     end
diff --git a/app/services/batched_remove_status_service.rb b/app/services/batched_remove_status_service.rb
index 6df8d4769..3638134be 100644
--- a/app/services/batched_remove_status_service.rb
+++ b/app/services/batched_remove_status_service.rb
@@ -8,7 +8,7 @@ class BatchedRemoveStatusService < BaseService
   # Dispatch Salmon deletes, unique per domain, of the deleted statuses, but only local ones
   # Remove statuses from home feeds
   # Push delete events to streaming API for home feeds and public feeds
-  # @param [Status] statuses A preferably batched array of statuses
+  # @param [Enumerable<Status>] statuses A preferably batched array of statuses
   # @param [Hash] options
   # @option [Boolean] :skip_side_effects
   def call(statuses, **options)
diff --git a/app/services/remove_status_service.rb b/app/services/remove_status_service.rb
index 91c934181..685c1d4bf 100644
--- a/app/services/remove_status_service.rb
+++ b/app/services/remove_status_service.rb
@@ -4,6 +4,11 @@ class RemoveStatusService < BaseService
   include Redisable
   include Payloadable
 
+  # Delete a status
+  # @param   [Status] status
+  # @param   [Hash] options
+  # @option  [Boolean] :redraft
+  # @options [Boolean] :original_removed
   def call(status, **options)
     @payload  = Oj.dump(event: :delete, payload: status.id.to_s)
     @status   = status
@@ -24,6 +29,7 @@ class RemoveStatusService < BaseService
         remove_from_public
         remove_from_media if status.media_attachments.any?
         remove_from_spam_check
+        remove_media
 
         @status.destroy!
       else
@@ -143,6 +149,12 @@ class RemoveStatusService < BaseService
     redis.publish('timeline:public:local:media', @payload) if @status.local?
   end
 
+  def remove_media
+    return if @options[:redraft]
+
+    @status.media_attachments.destroy_all
+  end
+
   def remove_from_spam_check
     redis.zremrangebyscore("spam_check:#{@status.account_id}", @status.id, @status.id)
   end
diff --git a/app/workers/removal_worker.rb b/app/workers/removal_worker.rb
index 19a660dd3..14423a4fb 100644
--- a/app/workers/removal_worker.rb
+++ b/app/workers/removal_worker.rb
@@ -3,8 +3,8 @@
 class RemovalWorker
   include Sidekiq::Worker
 
-  def perform(status_id)
-    RemoveStatusService.new.call(Status.find(status_id))
+  def perform(status_id, options = {})
+    RemoveStatusService.new.call(Status.find(status_id), **options.symbolize_keys)
   rescue ActiveRecord::RecordNotFound
     true
   end
diff --git a/spec/controllers/admin/reported_statuses_controller_spec.rb b/spec/controllers/admin/reported_statuses_controller_spec.rb
index c358506d6..bd146b795 100644
--- a/spec/controllers/admin/reported_statuses_controller_spec.rb
+++ b/spec/controllers/admin/reported_statuses_controller_spec.rb
@@ -47,7 +47,7 @@ describe Admin::ReportedStatusesController do
       it 'removes a status' do
         allow(RemovalWorker).to receive(:perform_async)
         subject.call
-        expect(RemovalWorker).to have_received(:perform_async).with(status_ids.first)
+        expect(RemovalWorker).to have_received(:perform_async).with(status_ids.first, redraft: false)
       end
     end
 
diff --git a/spec/controllers/admin/statuses_controller_spec.rb b/spec/controllers/admin/statuses_controller_spec.rb
index 1a08c10b7..6b06343ef 100644
--- a/spec/controllers/admin/statuses_controller_spec.rb
+++ b/spec/controllers/admin/statuses_controller_spec.rb
@@ -65,7 +65,7 @@ describe Admin::StatusesController do
       it 'removes a status' do
         allow(RemovalWorker).to receive(:perform_async)
         subject.call
-        expect(RemovalWorker).to have_received(:perform_async).with(status_ids.first)
+        expect(RemovalWorker).to have_received(:perform_async).with(status_ids.first, redraft: false)
       end
     end
 
diff --git a/spec/models/form/status_batch_spec.rb b/spec/models/form/status_batch_spec.rb
index 00c790a11..f9c58c90f 100644
--- a/spec/models/form/status_batch_spec.rb
+++ b/spec/models/form/status_batch_spec.rb
@@ -41,12 +41,12 @@ describe Form::StatusBatch do
 
     it 'call RemovalWorker' do
       form.save
-      expect(RemovalWorker).to have_received(:perform_async).with(status.id)
+      expect(RemovalWorker).to have_received(:perform_async).with(status.id, redraft: false)
     end
 
     it 'do not call RemovalWorker' do
       form.save
-      expect(RemovalWorker).not_to have_received(:perform_async).with(another_status.id)
+      expect(RemovalWorker).not_to have_received(:perform_async).with(another_status.id, redraft: false)
     end
   end
 end
-- 
cgit 


From 73ca0bb925cb036f824262ab292a157a40a515d0 Mon Sep 17 00:00:00 2001
From: Eugen Rochko <eugen@zeonfederated.com>
Date: Fri, 23 Aug 2019 22:37:23 +0200
Subject: Add option to include reported statuses in warning e-mail (#11639)

---
 .../admin/account_actions_controller.rb            |  4 ++--
 app/javascript/styles/mailer.scss                  |  7 ++++++
 app/mailers/user_mailer.rb                         |  4 +++-
 app/models/admin/account_action.rb                 | 22 +++++++++++++-----
 app/views/admin/account_actions/new.html.haml      |  4 ++++
 app/views/notification_mailer/_status.html.haml    |  8 ++++++-
 app/views/user_mailer/warning.html.haml            | 27 +++++++++++++++++++++-
 app/views/user_mailer/warning.text.erb             | 13 +++++++++++
 config/locales/en.yml                              |  2 ++
 config/locales/simple_form.en.yml                  |  2 ++
 spec/mailers/previews/user_mailer_preview.rb       |  2 +-
 spec/models/admin/account_action_spec.rb           |  4 ++--
 12 files changed, 85 insertions(+), 14 deletions(-)

(limited to 'spec')

diff --git a/app/controllers/admin/account_actions_controller.rb b/app/controllers/admin/account_actions_controller.rb
index a2cea461e..ea56fa0ac 100644
--- a/app/controllers/admin/account_actions_controller.rb
+++ b/app/controllers/admin/account_actions_controller.rb
@@ -5,7 +5,7 @@ module Admin
     before_action :set_account
 
     def new
-      @account_action  = Admin::AccountAction.new(type: params[:type], report_id: params[:report_id], send_email_notification: true)
+      @account_action  = Admin::AccountAction.new(type: params[:type], report_id: params[:report_id], send_email_notification: true, include_statuses: true)
       @warning_presets = AccountWarningPreset.all
     end
 
@@ -30,7 +30,7 @@ module Admin
     end
 
     def resource_params
-      params.require(:admin_account_action).permit(:type, :report_id, :warning_preset_id, :text, :send_email_notification)
+      params.require(:admin_account_action).permit(:type, :report_id, :warning_preset_id, :text, :send_email_notification, :include_statuses)
     end
   end
 end
diff --git a/app/javascript/styles/mailer.scss b/app/javascript/styles/mailer.scss
index b4fb1d709..e25a80c04 100644
--- a/app/javascript/styles/mailer.scss
+++ b/app/javascript/styles/mailer.scss
@@ -457,6 +457,13 @@ h5 {
 .status {
   padding-bottom: 32px;
 
+  &--highlighted {
+    border: 1px solid lighten($ui-base-color, 8%);
+    border-radius: 4px;
+    padding-bottom: 16px;
+    margin-bottom: 16px;
+  }
+
   .status-header {
     td {
       font-size: 14px;
diff --git a/app/mailers/user_mailer.rb b/app/mailers/user_mailer.rb
index 8f3a4ab3a..b41004acc 100644
--- a/app/mailers/user_mailer.rb
+++ b/app/mailers/user_mailer.rb
@@ -5,6 +5,7 @@ class UserMailer < Devise::Mailer
 
   helper :application
   helper :instance
+  helper :statuses
 
   add_template_helper RoutingHelper
 
@@ -79,10 +80,11 @@ class UserMailer < Devise::Mailer
     end
   end
 
-  def warning(user, warning)
+  def warning(user, warning, status_ids = nil)
     @resource = user
     @warning  = warning
     @instance = Rails.configuration.x.local_domain
+    @statuses = Status.where(id: status_ids).includes(:account) if status_ids.is_a?(Array)
 
     I18n.with_locale(@resource.locale || I18n.default_locale) do
       mail to: @resource.email,
diff --git a/app/models/admin/account_action.rb b/app/models/admin/account_action.rb
index bdbd342fb..c7da8b52c 100644
--- a/app/models/admin/account_action.rb
+++ b/app/models/admin/account_action.rb
@@ -19,20 +19,25 @@ class Admin::AccountAction
                 :report_id,
                 :warning_preset_id
 
-  attr_reader :warning, :send_email_notification
+  attr_reader :warning, :send_email_notification, :include_statuses
 
   def send_email_notification=(value)
     @send_email_notification = ActiveModel::Type::Boolean.new.cast(value)
   end
 
+  def include_statuses=(value)
+    @include_statuses = ActiveModel::Type::Boolean.new.cast(value)
+  end
+
   def save!
     ApplicationRecord.transaction do
       process_action!
       process_warning!
     end
 
-    queue_email!
+    process_email!
     process_reports!
+    process_queue!
   end
 
   def report
@@ -110,7 +115,6 @@ class Admin::AccountAction
     authorize(target_account, :suspend?)
     log_action(:suspend, target_account)
     target_account.suspend!
-    queue_suspension_worker!
   end
 
   def text_for_warning
@@ -121,16 +125,22 @@ class Admin::AccountAction
     Admin::SuspensionWorker.perform_async(target_account.id)
   end
 
-  def queue_email!
-    return unless warnable?
+  def process_queue!
+    queue_suspension_worker! if type == 'suspend'
+  end
 
-    UserMailer.warning(target_account.user, warning).deliver_later!
+  def process_email!
+    UserMailer.warning(target_account.user, warning, status_ids).deliver_now! if warnable?
   end
 
   def warnable?
     send_email_notification && target_account.local?
   end
 
+  def status_ids
+    @report.status_ids if @report && include_statuses
+  end
+
   def warning_preset
     @warning_preset ||= AccountWarningPreset.find(warning_preset_id) if warning_preset_id.present?
   end
diff --git a/app/views/admin/account_actions/new.html.haml b/app/views/admin/account_actions/new.html.haml
index 97286c8e5..20fbeef33 100644
--- a/app/views/admin/account_actions/new.html.haml
+++ b/app/views/admin/account_actions/new.html.haml
@@ -13,6 +13,10 @@
     .fields-group
       = f.input :send_email_notification, as: :boolean, wrapper: :with_label
 
+    - if params[:report_id].present?
+      .fields-group
+        = f.input :include_statuses, as: :boolean, wrapper: :with_label
+
     %hr.spacer/
 
     - unless @warning_presets.empty?
diff --git a/app/views/notification_mailer/_status.html.haml b/app/views/notification_mailer/_status.html.haml
index 57b5688bd..40f3aa88a 100644
--- a/app/views/notification_mailer/_status.html.haml
+++ b/app/views/notification_mailer/_status.html.haml
@@ -1,4 +1,5 @@
 - i ||= 0
+- highlighted ||= false
 
 %table.email-table{ cellspacing: 0, cellpadding: 0, dir: 'ltr' }
   %tbody
@@ -14,7 +15,7 @@
                       %table.column{ cellspacing: 0, cellpadding: 0 }
                         %tbody
                           %tr
-                            %td.column-cell.padded.status
+                            %td.column-cell.padded.status{ class: highlighted ? 'status--highlighted' : '' }
                               %table.status-header{ cellspacing: 0, cellpadding: 0 }
                                 %tbody
                                   %tr
@@ -32,5 +33,10 @@
                               %div{ dir: rtl_status?(status) ? 'rtl' : 'ltr' }
                                 = Formatter.instance.format(status)
 
+                                - if status.media_attachments.size > 0
+                                  %p
+                                    - status.media_attachments.each do |a|
+                                      = link_to medium_url(a), medium_url(a)
+
                               %p.status-footer
                                 = link_to l(status.created_at), web_url("statuses/#{status.id}")
diff --git a/app/views/user_mailer/warning.html.haml b/app/views/user_mailer/warning.html.haml
index 72ea5e5d2..030a57bb4 100644
--- a/app/views/user_mailer/warning.html.haml
+++ b/app/views/user_mailer/warning.html.haml
@@ -42,6 +42,14 @@
                               - unless @warning.text.blank?
                                 = Formatter.instance.linkify(@warning.text)
 
+                              - unless @statuses.empty?
+                                %p
+                                  %strong= t('user_mailer.warning.statuses')
+
+- unless @statuses.empty?
+  - @statuses.each_with_index do |status, i|
+    = render 'notification_mailer/status', status: status, i: i + 1, highlighted: true
+
 %table.email-table{ cellspacing: 0, cellpadding: 0 }
   %tbody
     %tr
@@ -50,7 +58,7 @@
           %table.content-section{ cellspacing: 0, cellpadding: 0 }
             %tbody
               %tr
-                %td.content-cell
+                %td.content-cell{ class: @statuses.empty? ? '' : 'content-start' }
                   %table.column{ cellspacing: 0, cellpadding: 0 }
                     %tbody
                       %tr
@@ -61,3 +69,20 @@
                                 %td.button-primary
                                   = link_to about_more_url do
                                     %span= t 'user_mailer.warning.review_server_policies'
+
+%table.email-table{ cellspacing: 0, cellpadding: 0 }
+  %tbody
+    %tr
+      %td.email-body
+        .email-container
+          %table.content-section{ cellspacing: 0, cellpadding: 0 }
+            %tbody
+              %tr
+                %td.content-cell
+                  .email-row
+                    .col-6
+                      %table.column{ cellspacing: 0, cellpadding: 0 }
+                        %tbody
+                          %tr
+                            %td.column-cell.text-center
+                              %p= t 'user_mailer.warning.get_in_touch', instance: @instance
diff --git a/app/views/user_mailer/warning.text.erb b/app/views/user_mailer/warning.text.erb
index b4f2402cb..24c1f86f2 100644
--- a/app/views/user_mailer/warning.text.erb
+++ b/app/views/user_mailer/warning.text.erb
@@ -7,3 +7,16 @@
 
 <% end %>
 <%= @warning.text %>
+<% unless @statuses.empty? %>
+<%= t('user_mailer.warning.statuses') %>
+
+<% @statuses.each do |status| %>
+
+<%= render 'notification_mailer/status', status: status %>
+---
+<% end %>
+<% else %>
+---
+<% end %>
+
+<%= t 'user_mailer.warning.get_in_touch', instance: @instance %>
diff --git a/config/locales/en.yml b/config/locales/en.yml
index a50dcb8a5..ee78e4720 100644
--- a/config/locales/en.yml
+++ b/config/locales/en.yml
@@ -1115,7 +1115,9 @@ en:
         disable: While your account is frozen, your account data remains intact, but you cannot perform any actions until it is unlocked.
         silence: While your account is limited, only people who are already following you will see your toots on this server, and you may be excluded from various public listings. However, others may still manually follow you.
         suspend: Your account has been suspended, and all of your toots and your uploaded media files have been irreversibly removed from this server, and servers where you had followers.
+      get_in_touch: You can reply to this e-mail to get in touch with the staff of %{instance}.
       review_server_policies: Review server policies
+      statuses: 'Specifically, for:'
       subject:
         disable: Your account %{acct} has been frozen
         none: Warning for %{acct}
diff --git a/config/locales/simple_form.en.yml b/config/locales/simple_form.en.yml
index 98f0843d0..cfaa6e666 100644
--- a/config/locales/simple_form.en.yml
+++ b/config/locales/simple_form.en.yml
@@ -5,6 +5,7 @@ en:
       account_warning_preset:
         text: You can use toot syntax, such as URLs, hashtags and mentions
       admin_account_action:
+        include_statuses: The user will see which toots have caused the moderation action or warning
         send_email_notification: The user will receive an explanation of what happened with their account
         text_html: Optional. You can use toot syntax. You can <a href="%{path}">add warning presets</a> to save time
         type_html: Choose what to do with <strong>%{acct}</strong>
@@ -60,6 +61,7 @@ en:
       account_warning_preset:
         text: Preset text
       admin_account_action:
+        include_statuses: Include reported toots in the e-mail
         send_email_notification: Notify the user per e-mail
         text: Custom warning
         type: Action
diff --git a/spec/mailers/previews/user_mailer_preview.rb b/spec/mailers/previews/user_mailer_preview.rb
index 53c836494..ead3b3baa 100644
--- a/spec/mailers/previews/user_mailer_preview.rb
+++ b/spec/mailers/previews/user_mailer_preview.rb
@@ -42,6 +42,6 @@ class UserMailerPreview < ActionMailer::Preview
 
   # Preview this email at http://localhost:3000/rails/mailers/user_mailer/warning
   def warning
-    UserMailer.warning(User.first, AccountWarning.new(text: '', action: :silence))
+    UserMailer.warning(User.first, AccountWarning.new(text: '', action: :silence), [Status.first.id])
   end
 end
diff --git a/spec/models/admin/account_action_spec.rb b/spec/models/admin/account_action_spec.rb
index a3db60cfc..87fc28500 100644
--- a/spec/models/admin/account_action_spec.rb
+++ b/spec/models/admin/account_action_spec.rb
@@ -58,8 +58,8 @@ RSpec.describe Admin::AccountAction, type: :model do
       end.to change { Admin::ActionLog.count }.by 1
     end
 
-    it 'calls queue_email!' do
-      expect(account_action).to receive(:queue_email!)
+    it 'calls process_email!' do
+      expect(account_action).to receive(:process_email!)
       subject
     end
 
-- 
cgit 


From 22ce4778eba300cdbd6a1eda94d49ce647ecdbaf Mon Sep 17 00:00:00 2001
From: Eugen Rochko <eugen@zeonfederated.com>
Date: Fri, 30 Aug 2019 01:34:47 +0200
Subject: Fix uncaught parameter missing exceptions and missing error templates
 (#11702)

---
 app/controllers/api/base_controller.rb                       |  8 ++++++++
 app/controllers/application_controller.rb                    | 12 +++++++++++-
 app/views/errors/400.html.haml                               |  5 +++++
 app/views/errors/406.html.haml                               |  5 +++++
 app/views/errors/503.html.haml                               |  5 +++++
 config/locales/en.yml                                        |  3 +++
 .../confirmations_controller_spec.rb                         |  3 ++-
 .../settings/two_factor_authentications_controller_spec.rb   |  3 ++-
 8 files changed, 41 insertions(+), 3 deletions(-)
 create mode 100644 app/views/errors/400.html.haml
 create mode 100644 app/views/errors/406.html.haml
 create mode 100644 app/views/errors/503.html.haml

(limited to 'spec')

diff --git a/app/controllers/api/base_controller.rb b/app/controllers/api/base_controller.rb
index de8fff30e..33df75b37 100644
--- a/app/controllers/api/base_controller.rb
+++ b/app/controllers/api/base_controller.rb
@@ -36,6 +36,14 @@ class Api::BaseController < ApplicationController
     render json: { error: 'This action is not allowed' }, status: 403
   end
 
+  rescue_from Mastodon::RaceConditionError do
+    render json: { error: 'There was a temporary problem serving your request, please try again' }, status: 503
+  end
+
+  rescue_from ActionController::ParameterMissing do |e|
+    render json: { error: e.to_s }, status: 400
+  end
+
   def doorkeeper_unauthorized_render_options(error: nil)
     { json: { error: (error.try(:description) || 'Not authorized') } }
   end
diff --git a/app/controllers/application_controller.rb b/app/controllers/application_controller.rb
index 1caaa20f7..5b343a276 100644
--- a/app/controllers/application_controller.rb
+++ b/app/controllers/application_controller.rb
@@ -21,11 +21,13 @@ class ApplicationController < ActionController::Base
   helper_method :whitelist_mode?
 
   rescue_from ActionController::RoutingError, with: :not_found
-  rescue_from ActiveRecord::RecordNotFound, with: :not_found
   rescue_from ActionController::InvalidAuthenticityToken, with: :unprocessable_entity
   rescue_from ActionController::UnknownFormat, with: :not_acceptable
+  rescue_from ActionController::ParameterMissing, with: :bad_request
+  rescue_from ActiveRecord::RecordNotFound, with: :not_found
   rescue_from Mastodon::NotPermittedError, with: :forbidden
   rescue_from HTTP::Error, OpenSSL::SSL::SSLError, with: :internal_server_error
+  rescue_from Mastodon::RaceConditionError, with: :service_unavailable
 
   before_action :store_current_location, except: :raise_not_found, unless: :devise_controller?
   before_action :require_functional!, if: :user_signed_in?
@@ -96,10 +98,18 @@ class ApplicationController < ActionController::Base
     respond_with_error(406)
   end
 
+  def bad_request
+    respond_with_error(400)
+  end
+
   def internal_server_error
     respond_with_error(500)
   end
 
+  def service_unavailable
+    respond_with_error(503)
+  end
+
   def single_user_mode?
     @single_user_mode ||= Rails.configuration.x.single_user_mode && Account.where('id > 0').exists?
   end
diff --git a/app/views/errors/400.html.haml b/app/views/errors/400.html.haml
new file mode 100644
index 000000000..11fbdd40c
--- /dev/null
+++ b/app/views/errors/400.html.haml
@@ -0,0 +1,5 @@
+- content_for :page_title do
+  = t('errors.400')
+
+- content_for :content do
+  = t('errors.400')
diff --git a/app/views/errors/406.html.haml b/app/views/errors/406.html.haml
new file mode 100644
index 000000000..0ef815df3
--- /dev/null
+++ b/app/views/errors/406.html.haml
@@ -0,0 +1,5 @@
+- content_for :page_title do
+  = t('errors.406')
+
+- content_for :content do
+  = t('errors.406')
diff --git a/app/views/errors/503.html.haml b/app/views/errors/503.html.haml
new file mode 100644
index 000000000..b0c895aa5
--- /dev/null
+++ b/app/views/errors/503.html.haml
@@ -0,0 +1,5 @@
+- content_for :page_title do
+  = t('errors.503')
+
+- content_for :content do
+  = t('errors.503')
diff --git a/config/locales/en.yml b/config/locales/en.yml
index 2f601f274..892d13c72 100644
--- a/config/locales/en.yml
+++ b/config/locales/en.yml
@@ -652,8 +652,10 @@ en:
   domain_validator:
     invalid_domain: is not a valid domain name
   errors:
+    '400': The request you submitted was invalid or malformed.
     '403': You don't have permission to view this page.
     '404': The page you are looking for isn't here.
+    '406': This page is not available in the requested format.
     '410': The page you were looking for doesn't exist here anymore.
     '422':
       content: Security verification failed. Are you blocking cookies?
@@ -662,6 +664,7 @@ en:
     '500':
       content: We're sorry, but something went wrong on our end.
       title: This page is not correct
+    '503': The page could not be served due to a temporary server failure.
     noscript_html: To use the Mastodon web application, please enable JavaScript. Alternatively, try one of the <a href="%{apps_path}">native apps</a> for Mastodon for your platform.
   existing_username_validator:
     not_found: could not find a local user with that username
diff --git a/spec/controllers/settings/two_factor_authentication/confirmations_controller_spec.rb b/spec/controllers/settings/two_factor_authentication/confirmations_controller_spec.rb
index 478f24585..2222a7559 100644
--- a/spec/controllers/settings/two_factor_authentication/confirmations_controller_spec.rb
+++ b/spec/controllers/settings/two_factor_authentication/confirmations_controller_spec.rb
@@ -50,7 +50,8 @@ describe Settings::TwoFactorAuthentication::ConfirmationsController do
 
       describe 'when form_two_factor_confirmation parameter is not provided' do
         it 'raises ActionController::ParameterMissing' do
-          expect { post :create, params: {} }.to raise_error(ActionController::ParameterMissing)
+          post :create, params: {}
+          expect(response).to have_http_status(400)
         end
       end
 
diff --git a/spec/controllers/settings/two_factor_authentications_controller_spec.rb b/spec/controllers/settings/two_factor_authentications_controller_spec.rb
index 9f27222ad..f7c628756 100644
--- a/spec/controllers/settings/two_factor_authentications_controller_spec.rb
+++ b/spec/controllers/settings/two_factor_authentications_controller_spec.rb
@@ -112,7 +112,8 @@ describe Settings::TwoFactorAuthenticationsController do
       end
 
       it 'raises ActionController::ParameterMissing if code is missing' do
-        expect { post :destroy }.to raise_error(ActionController::ParameterMissing)
+        post :destroy
+        expect(response).to have_http_status(400)
       end
     end
 
-- 
cgit 


From b54b725d6bc8dd0a4ab0fe0bf408193c1bae8106 Mon Sep 17 00:00:00 2001
From: Eugen Rochko <eugen@zeonfederated.com>
Date: Fri, 30 Aug 2019 02:19:17 +0200
Subject: Fix uncaught domain normalization error in remote follow (#11703)

---
 app/controllers/remote_follow_controller.rb       |  2 +-
 app/controllers/remote_interaction_controller.rb  |  2 +-
 app/models/remote_follow.rb                       |  6 ++++--
 app/validators/domain_validator.rb                | 12 ++++++++++--
 spec/controllers/remote_follow_controller_spec.rb |  4 +---
 spec/models/remote_follow_spec.rb                 |  2 +-
 6 files changed, 18 insertions(+), 10 deletions(-)

(limited to 'spec')

diff --git a/app/controllers/remote_follow_controller.rb b/app/controllers/remote_follow_controller.rb
index 0fb71d335..ba963a7a0 100644
--- a/app/controllers/remote_follow_controller.rb
+++ b/app/controllers/remote_follow_controller.rb
@@ -29,7 +29,7 @@ class RemoteFollowController < ApplicationController
   end
 
   def session_params
-    { acct: session[:remote_follow] }
+    { acct: session[:remote_follow] || current_account&.username }
   end
 
   def set_body_classes
diff --git a/app/controllers/remote_interaction_controller.rb b/app/controllers/remote_interaction_controller.rb
index de5616e25..15224e853 100644
--- a/app/controllers/remote_interaction_controller.rb
+++ b/app/controllers/remote_interaction_controller.rb
@@ -32,7 +32,7 @@ class RemoteInteractionController < ApplicationController
   end
 
   def session_params
-    { acct: session[:remote_follow] }
+    { acct: session[:remote_follow] || current_account&.username }
   end
 
   def set_status
diff --git a/app/models/remote_follow.rb b/app/models/remote_follow.rb
index 93df11724..52dd3f67b 100644
--- a/app/models/remote_follow.rb
+++ b/app/models/remote_follow.rb
@@ -6,7 +6,7 @@ class RemoteFollow
 
   attr_accessor :acct, :addressable_template
 
-  validates :acct, presence: true
+  validates :acct, presence: true, domain: { acct: true }
 
   def initialize(attrs = {})
     @acct = normalize_acct(attrs[:acct])
@@ -21,7 +21,7 @@ class RemoteFollow
   end
 
   def subscribe_address_for(account)
-    addressable_template.expand(uri: account.local_username_and_domain).to_s
+    addressable_template.expand(uri: ActivityPub::TagManager.instance.uri_for(account)).to_s
   end
 
   def interact_address_for(status)
@@ -44,6 +44,8 @@ class RemoteFollow
     end
 
     [username, domain].compact.join('@')
+  rescue Addressable::URI::InvalidURIError
+    value
   end
 
   def fetch_template!
diff --git a/app/validators/domain_validator.rb b/app/validators/domain_validator.rb
index ae07f1798..6e4a854ff 100644
--- a/app/validators/domain_validator.rb
+++ b/app/validators/domain_validator.rb
@@ -4,14 +4,22 @@ class DomainValidator < ActiveModel::EachValidator
   def validate_each(record, attribute, value)
     return if value.blank?
 
-    record.errors.add(attribute, I18n.t('domain_validator.invalid_domain')) unless compliant?(value)
+    domain = begin
+      if options[:acct]
+        value.split('@').last
+      else
+        value
+      end
+    end
+
+    record.errors.add(attribute, I18n.t('domain_validator.invalid_domain')) unless compliant?(domain)
   end
 
   private
 
   def compliant?(value)
     Addressable::URI.new.tap { |uri| uri.host = value }
-  rescue Addressable::URI::InvalidURIError
+  rescue Addressable::URI::InvalidURIError, IDN::Idna::IdnaError
     false
   end
 end
diff --git a/spec/controllers/remote_follow_controller_spec.rb b/spec/controllers/remote_follow_controller_spec.rb
index 5088c2e65..d79dd2949 100644
--- a/spec/controllers/remote_follow_controller_spec.rb
+++ b/spec/controllers/remote_follow_controller_spec.rb
@@ -66,9 +66,7 @@ describe RemoteFollowController do
         end
 
         it 'redirects to the remote location' do
-          address = "http://example.com/follow_me?acct=test_user%40#{Rails.configuration.x.local_domain}"
-
-          expect(response).to redirect_to(address)
+          expect(response).to redirect_to("http://example.com/follow_me?acct=https%3A%2F%2F#{Rails.configuration.x.local_domain}%2Fusers%2Ftest_user")
         end
       end
     end
diff --git a/spec/models/remote_follow_spec.rb b/spec/models/remote_follow_spec.rb
index ed2667b28..5b4c19b5b 100644
--- a/spec/models/remote_follow_spec.rb
+++ b/spec/models/remote_follow_spec.rb
@@ -61,7 +61,7 @@ RSpec.describe RemoteFollow do
     subject { remote_follow.subscribe_address_for(account) }
 
     it 'returns subscribe address' do
-      is_expected.to eq 'https://quitter.no/main/ostatussub?profile=alice%40cb6e6126.ngrok.io'
+      is_expected.to eq 'https://quitter.no/main/ostatussub?profile=https%3A%2F%2Fcb6e6126.ngrok.io%2Fusers%2Falice'
     end
   end
 end
-- 
cgit 


From 70ddef2654a931827ce5e4323e3042365f6078f2 Mon Sep 17 00:00:00 2001
From: Eugen Rochko <eugen@zeonfederated.com>
Date: Mon, 2 Sep 2019 18:11:13 +0200
Subject: Change trending hashtags to not disappear instantly after midnight
 (#11712)

---
 app/controllers/admin/tags_controller.rb           |   2 +-
 app/lib/feed_manager.rb                            |   2 +-
 app/models/tag.rb                                  |   4 +-
 app/models/trending_tags.rb                        | 102 +++++++++++++++------
 app/workers/scheduler/trending_tags_scheduler.rb   |  11 +++
 config/sidekiq.yml                                 |   3 +
 db/migrate/20190901035623_add_max_score_to_tags.rb |   6 ++
 .../20190901040524_remove_score_from_tags.rb       |  12 +++
 db/schema.rb                                       |   6 +-
 spec/models/trending_tags_spec.rb                  |  68 ++++++++++++++
 10 files changed, 179 insertions(+), 37 deletions(-)
 create mode 100644 app/workers/scheduler/trending_tags_scheduler.rb
 create mode 100644 db/migrate/20190901035623_add_max_score_to_tags.rb
 create mode 100644 db/post_migrate/20190901040524_remove_score_from_tags.rb
 create mode 100644 spec/models/trending_tags_spec.rb

(limited to 'spec')

diff --git a/app/controllers/admin/tags_controller.rb b/app/controllers/admin/tags_controller.rb
index 25d9b7d3d..8bd4e5f8b 100644
--- a/app/controllers/admin/tags_controller.rb
+++ b/app/controllers/admin/tags_controller.rb
@@ -57,7 +57,7 @@ module Admin
       scope = scope.unreviewed if filter_params[:review] == 'unreviewed'
       scope = scope.reviewed.order(reviewed_at: :desc) if filter_params[:review] == 'reviewed'
       scope = scope.pending_review.order(requested_review_at: :desc) if filter_params[:review] == 'pending_review'
-      scope.order(score: :desc)
+      scope.order(max_score: :desc)
     end
 
     def filter_params
diff --git a/app/lib/feed_manager.rb b/app/lib/feed_manager.rb
index ca3d890a8..871ec5c19 100644
--- a/app/lib/feed_manager.rb
+++ b/app/lib/feed_manager.rb
@@ -63,7 +63,7 @@ class FeedManager
     reblog_key   = key(type, account_id, 'reblogs')
 
     # Remove any items past the MAX_ITEMS'th entry in our feed
-    redis.zremrangebyrank(timeline_key, '0', (-(FeedManager::MAX_ITEMS + 1)).to_s)
+    redis.zremrangebyrank(timeline_key, 0, -(FeedManager::MAX_ITEMS + 1))
 
     # Get the score of the REBLOG_FALLOFF'th item in our feed, and stop
     # tracking anything after it for deduplication purposes.
diff --git a/app/models/tag.rb b/app/models/tag.rb
index 945e3a3c6..135e0a030 100644
--- a/app/models/tag.rb
+++ b/app/models/tag.rb
@@ -7,14 +7,14 @@
 #  name                :string           default(""), not null
 #  created_at          :datetime         not null
 #  updated_at          :datetime         not null
-#  score               :integer
 #  usable              :boolean
 #  trendable           :boolean
 #  listable            :boolean
 #  reviewed_at         :datetime
 #  requested_review_at :datetime
 #  last_status_at      :datetime
-#  last_trend_at       :datetime
+#  max_score           :float
+#  max_score_at        :datetime
 #
 
 class Tag < ApplicationRecord
diff --git a/app/models/trending_tags.rb b/app/models/trending_tags.rb
index e4ce988c1..e1b92b175 100644
--- a/app/models/trending_tags.rb
+++ b/app/models/trending_tags.rb
@@ -7,6 +7,8 @@ class TrendingTags
   THRESHOLD            = 5
   LIMIT                = 10
   REVIEW_THRESHOLD     = 3
+  MAX_SCORE_COOLDOWN   = 3.days.freeze
+  MAX_SCORE_HALFLIFE   = 6.hours.freeze
 
   class << self
     include Redisable
@@ -16,14 +18,75 @@ class TrendingTags
 
       increment_historical_use!(tag.id, at_time)
       increment_unique_use!(tag.id, account.id, at_time)
-      increment_vote!(tag, at_time)
+      increment_use!(tag.id, at_time)
 
       tag.update(last_status_at: Time.now.utc) if tag.last_status_at.nil? || tag.last_status_at < 12.hours.ago
-      tag.update(last_trend_at: Time.now.utc)  if trending?(tag) && (tag.last_trend_at.nil? || tag.last_trend_at < 12.hours.ago)
+    end
+
+    def update!(at_time = Time.now.utc)
+      tag_ids = redis.smembers("#{KEY}:used:#{at_time.beginning_of_day.to_i}") + redis.zrange(KEY, 0, -1)
+      tags    = Tag.where(id: tag_ids.uniq)
+
+      # First pass to calculate scores and update the set
+
+      tags.each do |tag|
+        expected  = redis.pfcount("activity:tags:#{tag.id}:#{(at_time - 1.day).beginning_of_day.to_i}:accounts").to_f
+        expected  = 1.0 if expected.zero?
+        observed  = redis.pfcount("activity:tags:#{tag.id}:#{at_time.beginning_of_day.to_i}:accounts").to_f
+        max_time  = tag.max_score_at
+        max_score = tag.max_score
+        max_score = 0 if max_time.nil? || max_time < (at_time - MAX_SCORE_COOLDOWN)
+
+        score = begin
+          if expected > observed || observed < THRESHOLD
+            0
+          else
+            ((observed - expected)**2) / expected
+          end
+        end
+
+        if score > max_score
+          max_score = score
+          max_time  = at_time
+
+          # Not interested in triggering any callbacks for this
+          tag.update_columns(max_score: max_score, max_score_at: max_time)
+        end
+
+        decaying_score = max_score * (0.5**((at_time.to_f - max_time.to_f) / MAX_SCORE_HALFLIFE.to_f))
+
+        if decaying_score.zero?
+          redis.zrem(KEY, tag.id)
+        else
+          redis.zadd(KEY, decaying_score, tag.id)
+        end
+      end
+
+      users_for_review = User.staff.includes(:account).to_a.select(&:allows_trending_tag_emails?)
+
+      # Second pass to notify about previously unreviewed trends
+
+      tags.each do |tag|
+        current_rank              = redis.zrevrank(KEY, tag.id)
+        needs_review_notification = tag.requires_review? && !tag.requested_review?
+        rank_passes_threshold     = current_rank.present? && current_rank <= REVIEW_THRESHOLD
+
+        next unless !tag.trendable? && rank_passes_threshold && needs_review_notification
+
+        tag.touch(:requested_review_at)
+
+        users_for_review.each do |user|
+          AdminMailer.new_trending_tag(user.account, tag).deliver_later!
+        end
+      end
+
+      # Trim older items
+
+      redis.zremrangebyrank(KEY, 0, -(LIMIT + 1))
     end
 
     def get(limit, filtered: true)
-      tag_ids = redis.zrevrange("#{KEY}:#{Time.now.utc.beginning_of_day.to_i}", 0, LIMIT - 1).map(&:to_i)
+      tag_ids = redis.zrevrange(KEY, 0, LIMIT - 1).map(&:to_i)
 
       tags = Tag.where(id: tag_ids)
       tags = tags.where(trendable: true) if filtered
@@ -33,8 +96,8 @@ class TrendingTags
     end
 
     def trending?(tag)
-      rank = redis.zrevrank("#{KEY}:#{Time.now.utc.beginning_of_day.to_i}", tag.id)
-      rank.present? && rank <= LIMIT
+      rank = redis.zrevrank(KEY, tag.id)
+      rank.present? && rank < LIMIT
     end
 
     private
@@ -51,31 +114,10 @@ class TrendingTags
       redis.expire(key, EXPIRE_HISTORY_AFTER)
     end
 
-    def increment_vote!(tag, at_time)
-      key      = "#{KEY}:#{at_time.beginning_of_day.to_i}"
-      expected = redis.pfcount("activity:tags:#{tag.id}:#{(at_time - 1.day).beginning_of_day.to_i}:accounts").to_f
-      expected = 1.0 if expected.zero?
-      observed = redis.pfcount("activity:tags:#{tag.id}:#{at_time.beginning_of_day.to_i}:accounts").to_f
-
-      if expected > observed || observed < THRESHOLD
-        redis.zrem(key, tag.id)
-      else
-        score    = ((observed - expected)**2) / expected
-        old_rank = redis.zrevrank(key, tag.id)
-
-        redis.zadd(key, score, tag.id)
-        request_review!(tag) if (old_rank.nil? || old_rank > REVIEW_THRESHOLD) && redis.zrevrank(key, tag.id) <= REVIEW_THRESHOLD && !tag.trendable? && tag.requires_review? && !tag.requested_review?
-      end
-
-      redis.expire(key, EXPIRE_TRENDS_AFTER)
-    end
-
-    def request_review!(tag)
-      return unless Setting.trends
-
-      tag.touch(:requested_review_at)
-
-      User.staff.includes(:account).find_each { |u| AdminMailer.new_trending_tag(u.account, tag).deliver_later! if u.allows_trending_tag_emails? }
+    def increment_use!(tag_id, at_time)
+      key = "#{KEY}:used:#{at_time.beginning_of_day.to_i}"
+      redis.sadd(key, tag_id)
+      redis.expire(key, EXPIRE_HISTORY_AFTER)
     end
   end
 end
diff --git a/app/workers/scheduler/trending_tags_scheduler.rb b/app/workers/scheduler/trending_tags_scheduler.rb
new file mode 100644
index 000000000..77f0d5747
--- /dev/null
+++ b/app/workers/scheduler/trending_tags_scheduler.rb
@@ -0,0 +1,11 @@
+# frozen_string_literal: true
+
+class Scheduler::TrendingTagsScheduler
+  include Sidekiq::Worker
+
+  sidekiq_options unique: :until_executed, retry: 0
+
+  def perform
+    TrendingTags.update! if Setting.trends
+  end
+end
diff --git a/config/sidekiq.yml b/config/sidekiq.yml
index 6ebe450b0..5de25de23 100644
--- a/config/sidekiq.yml
+++ b/config/sidekiq.yml
@@ -9,6 +9,9 @@
   scheduled_statuses_scheduler:
     every: '5m'
     class: Scheduler::ScheduledStatusesScheduler
+  trending_tags_scheduler:
+    every: '5m'
+    class: Scheduler::TrendingTagsScheduler
   media_cleanup_scheduler:
     cron: '<%= Random.rand(0..59) %> <%= Random.rand(3..5) %> * * *'
     class: Scheduler::MediaCleanupScheduler
diff --git a/db/migrate/20190901035623_add_max_score_to_tags.rb b/db/migrate/20190901035623_add_max_score_to_tags.rb
new file mode 100644
index 000000000..f936e9871
--- /dev/null
+++ b/db/migrate/20190901035623_add_max_score_to_tags.rb
@@ -0,0 +1,6 @@
+class AddMaxScoreToTags < ActiveRecord::Migration[5.2]
+  def change
+    add_column :tags, :max_score, :float
+    add_column :tags, :max_score_at, :datetime
+  end
+end
diff --git a/db/post_migrate/20190901040524_remove_score_from_tags.rb b/db/post_migrate/20190901040524_remove_score_from_tags.rb
new file mode 100644
index 000000000..a1112700b
--- /dev/null
+++ b/db/post_migrate/20190901040524_remove_score_from_tags.rb
@@ -0,0 +1,12 @@
+# frozen_string_literal: true
+
+class RemoveScoreFromTags < ActiveRecord::Migration[5.2]
+  disable_ddl_transaction!
+
+  def change
+    safety_assured do
+      remove_column :tags, :score, :int
+      remove_column :tags, :last_trend_at, :datetime
+    end
+  end
+end
diff --git a/db/schema.rb b/db/schema.rb
index 482bca367..5576f70bf 100644
--- a/db/schema.rb
+++ b/db/schema.rb
@@ -10,7 +10,7 @@
 #
 # It's strongly recommended that you check this file into your version control system.
 
-ActiveRecord::Schema.define(version: 2019_08_23_221802) do
+ActiveRecord::Schema.define(version: 2019_09_01_040524) do
 
   # These are extensions that must be enabled in order to support this database
   enable_extension "plpgsql"
@@ -664,14 +664,14 @@ ActiveRecord::Schema.define(version: 2019_08_23_221802) do
     t.string "name", default: "", null: false
     t.datetime "created_at", null: false
     t.datetime "updated_at", null: false
-    t.integer "score"
     t.boolean "usable"
     t.boolean "trendable"
     t.boolean "listable"
     t.datetime "reviewed_at"
     t.datetime "requested_review_at"
     t.datetime "last_status_at"
-    t.datetime "last_trend_at"
+    t.float "max_score"
+    t.datetime "max_score_at"
     t.index "lower((name)::text)", name: "index_tags_on_name_lower", unique: true
   end
 
diff --git a/spec/models/trending_tags_spec.rb b/spec/models/trending_tags_spec.rb
new file mode 100644
index 000000000..b6122c994
--- /dev/null
+++ b/spec/models/trending_tags_spec.rb
@@ -0,0 +1,68 @@
+require 'rails_helper'
+
+RSpec.describe TrendingTags do
+  describe '.record_use!' do
+    pending
+  end
+
+  describe '.update!' do
+    let!(:at_time) { Time.now.utc }
+    let!(:tag1) { Fabricate(:tag, name: 'Catstodon') }
+    let!(:tag2) { Fabricate(:tag, name: 'DogsOfMastodon') }
+    let!(:tag3) { Fabricate(:tag, name: 'OCs') }
+
+    before do
+      allow(Redis.current).to receive(:pfcount) do |key|
+        case key
+        when "activity:tags:#{tag1.id}:#{(at_time - 1.day).beginning_of_day.to_i}:accounts"
+          2
+        when "activity:tags:#{tag1.id}:#{at_time.beginning_of_day.to_i}:accounts"
+          16
+        when "activity:tags:#{tag2.id}:#{(at_time - 1.day).beginning_of_day.to_i}:accounts"
+          0
+        when "activity:tags:#{tag2.id}:#{at_time.beginning_of_day.to_i}:accounts"
+          4
+        when "activity:tags:#{tag3.id}:#{(at_time - 1.day).beginning_of_day.to_i}:accounts"
+          13
+        end
+      end
+
+      Redis.current.zadd('trending_tags', 0.9, tag3.id)
+      Redis.current.sadd("trending_tags:used:#{at_time.beginning_of_day.to_i}", [tag1.id, tag2.id])
+
+      tag3.update(max_score: 0.9, max_score_at: (at_time - 1.day).beginning_of_day + 12.hours)
+
+      described_class.update!(at_time)
+    end
+
+    it 'calculates and re-calculates scores' do
+      expect(described_class.get(10, filtered: false)).to eq [tag1, tag3]
+    end
+
+    it 'omits hashtags below threshold' do
+      expect(described_class.get(10, filtered: false)).to_not include(tag2)
+    end
+
+    it 'decays scores' do
+      expect(Redis.current.zscore('trending_tags', tag3.id)).to be < 0.9
+    end
+  end
+
+  describe '.trending?' do
+    let(:tag) { Fabricate(:tag) }
+
+    before do
+      10.times { |i| Redis.current.zadd('trending_tags', i + 1, Fabricate(:tag).id) }
+    end
+
+    it 'returns true if the hashtag is within limit' do
+      Redis.current.zadd('trending_tags', 11, tag.id)
+      expect(described_class.trending?(tag)).to be true
+    end
+
+    it 'returns false if the hashtag is outside the limit' do
+      Redis.current.zadd('trending_tags', 0, tag.id)
+      expect(described_class.trending?(tag)).to be false
+    end
+  end
+end
-- 
cgit 


From 692c5b439ae8659e459da692cf9e6b8e6f29d2a1 Mon Sep 17 00:00:00 2001
From: ThibG <thib@sitedethib.com>
Date: Tue, 3 Sep 2019 22:52:32 +0200
Subject: Fix ActivityPub context not being dynamically computed (#11746)

* Fix contexts not being dynamically included

Fixes #11649

* Refactor Note context in serializer

* Refactor Actor serializer
---
 app/lib/activitypub/adapter.rb                  | 13 +++++++------
 app/lib/activitypub/serializer.rb               |  8 ++++++++
 app/serializers/activitypub/actor_serializer.rb |  4 +++-
 app/serializers/activitypub/note_serializer.rb  |  7 +++++--
 config/initializers/active_model_serializers.rb | 19 -------------------
 spec/lib/activitypub/activity/update_spec.rb    |  2 +-
 6 files changed, 24 insertions(+), 29 deletions(-)

(limited to 'spec')

diff --git a/app/lib/activitypub/adapter.rb b/app/lib/activitypub/adapter.rb
index 1c58be8c0..cb2ac72d4 100644
--- a/app/lib/activitypub/adapter.rb
+++ b/app/lib/activitypub/adapter.rb
@@ -32,22 +32,23 @@ class ActivityPub::Adapter < ActiveModelSerializers::Adapter::Base
   end
 
   def serializable_hash(options = nil)
+    named_contexts     = {}
+    context_extensions = {}
     options         = serialization_options(options)
-    serialized_hash = serializer.serializable_hash(options)
+    serialized_hash = serializer.serializable_hash(options.merge(named_contexts: named_contexts, context_extensions: context_extensions))
     serialized_hash = serialized_hash.select { |k, _| options[:fields].include?(k) } if options[:fields]
     serialized_hash = self.class.transform_key_casing!(serialized_hash, instance_options)
 
-    { '@context' => serialized_context }.merge(serialized_hash)
+    { '@context' => serialized_context(named_contexts, context_extensions) }.merge(serialized_hash)
   end
 
   private
 
-  def serialized_context
+  def serialized_context(named_contexts_map, context_extensions_map)
     context_array = []
 
-    serializer_options = serializer.send(:instance_options) || {}
-    named_contexts     = [:activitystreams] + serializer._named_contexts.keys + serializer_options.fetch(:named_contexts, {}).keys
-    context_extensions = serializer._context_extensions.keys + serializer_options.fetch(:context_extensions, {}).keys
+    named_contexts     = [:activitystreams] + named_contexts_map.keys
+    context_extensions = context_extensions_map.keys
 
     named_contexts.each do |key|
       context_array << NAMED_CONTEXT_MAP[key]
diff --git a/app/lib/activitypub/serializer.rb b/app/lib/activitypub/serializer.rb
index 07bd8c494..1fdc79310 100644
--- a/app/lib/activitypub/serializer.rb
+++ b/app/lib/activitypub/serializer.rb
@@ -27,4 +27,12 @@ class ActivityPub::Serializer < ActiveModel::Serializer
       _context_extensions[extension_name] = true
     end
   end
+
+  def serializable_hash(adapter_options = nil, options = {}, adapter_instance = self.class.serialization_adapter_instance)
+    unless adapter_options&.fetch(:named_contexts, nil).nil?
+      adapter_options[:named_contexts].merge!(_named_contexts)
+      adapter_options[:context_extensions].merge!(_context_extensions)
+    end
+    super(adapter_options, options, adapter_instance)
+  end
 end
diff --git a/app/serializers/activitypub/actor_serializer.rb b/app/serializers/activitypub/actor_serializer.rb
index 222e17c99..17df85de3 100644
--- a/app/serializers/activitypub/actor_serializer.rb
+++ b/app/serializers/activitypub/actor_serializer.rb
@@ -6,7 +6,7 @@ class ActivityPub::ActorSerializer < ActivityPub::Serializer
   context :security
 
   context_extensions :manually_approves_followers, :featured, :also_known_as,
-                     :moved_to, :property_value, :hashtag, :emoji, :identity_proof,
+                     :moved_to, :property_value, :identity_proof,
                      :discoverable
 
   attributes :id, :type, :following, :followers,
@@ -138,6 +138,8 @@ class ActivityPub::ActorSerializer < ActivityPub::Serializer
   end
 
   class TagSerializer < ActivityPub::Serializer
+    context_extensions :hashtag
+
     include RoutingHelper
 
     attributes :type, :href, :name
diff --git a/app/serializers/activitypub/note_serializer.rb b/app/serializers/activitypub/note_serializer.rb
index 7592e0b1a..364d3eda5 100644
--- a/app/serializers/activitypub/note_serializer.rb
+++ b/app/serializers/activitypub/note_serializer.rb
@@ -1,8 +1,7 @@
 # frozen_string_literal: true
 
 class ActivityPub::NoteSerializer < ActivityPub::Serializer
-  context_extensions :atom_uri, :conversation, :sensitive,
-                     :hashtag, :emoji, :focal_point, :blurhash
+  context_extensions :atom_uri, :conversation, :sensitive
 
   attributes :id, :type, :summary,
              :in_reply_to, :published, :url,
@@ -151,6 +150,8 @@ class ActivityPub::NoteSerializer < ActivityPub::Serializer
   end
 
   class MediaAttachmentSerializer < ActivityPub::Serializer
+    context_extensions :blurhash, :focal_point
+
     include RoutingHelper
 
     attributes :type, :media_type, :url, :name, :blurhash
@@ -198,6 +199,8 @@ class ActivityPub::NoteSerializer < ActivityPub::Serializer
   end
 
   class TagSerializer < ActivityPub::Serializer
+    context_extensions :hashtag
+
     include RoutingHelper
 
     attributes :type, :href, :name
diff --git a/config/initializers/active_model_serializers.rb b/config/initializers/active_model_serializers.rb
index 329a5fb2c..0e69e1d96 100644
--- a/config/initializers/active_model_serializers.rb
+++ b/config/initializers/active_model_serializers.rb
@@ -3,22 +3,3 @@ ActiveModelSerializers.config.tap do |config|
 end
 
 ActiveSupport::Notifications.unsubscribe(ActiveModelSerializers::Logging::RENDER_EVENT)
-
-class ActiveModel::Serializer::Reflection
-  # We monkey-patch this method so that when we include associations in a serializer,
-  # the nested serializers can send information about used contexts upwards back to
-  # the root. We do this via instance_options because the nesting can be dynamic.
-  def build_association(parent_serializer, parent_serializer_options, include_slice = {})
-    serializer = options[:serializer]
-
-    parent_serializer_options.merge!(named_contexts: serializer._named_contexts, context_extensions: serializer._context_extensions) if serializer.respond_to?(:_named_contexts)
-
-    association_options = {
-      parent_serializer: parent_serializer,
-      parent_serializer_options: parent_serializer_options,
-      include_slice: include_slice,
-    }
-
-    ActiveModel::Serializer::Association.new(self, association_options)
-  end
-end
diff --git a/spec/lib/activitypub/activity/update_spec.rb b/spec/lib/activitypub/activity/update_spec.rb
index fbfc585cf..42da29860 100644
--- a/spec/lib/activitypub/activity/update_spec.rb
+++ b/spec/lib/activitypub/activity/update_spec.rb
@@ -19,7 +19,7 @@ RSpec.describe ActivityPub::Activity::Update do
   end
 
   let(:actor_json) do
-    ActiveModelSerializers::SerializableResource.new(modified_sender, serializer: ActivityPub::ActorSerializer, key_transform: :camel_lower).as_json
+    ActiveModelSerializers::SerializableResource.new(modified_sender, serializer: ActivityPub::ActorSerializer, adapter: ActivityPub::Adapter).as_json
   end
 
   let(:json) do
-- 
cgit 


From e445a8af64908b2bdb721bec74c113e8258a129b Mon Sep 17 00:00:00 2001
From: Eugen Rochko <eugen@zeonfederated.com>
Date: Fri, 6 Sep 2019 13:55:51 +0200
Subject: Add timeline read markers API (#11762)

Fix #4093
---
 app/controllers/api/v1/markers_controller.rb       | 44 +++++++++++++++
 app/javascript/mastodon/actions/markers.js         | 30 ++++++++++
 app/javascript/mastodon/features/ui/index.js       |  5 +-
 app/models/marker.rb                               | 23 ++++++++
 app/models/user.rb                                 |  1 +
 app/serializers/rest/marker_serializer.rb          | 13 +++++
 config/routes.rb                                   |  1 +
 db/migrate/20190904222339_create_markers.rb        | 14 +++++
 db/schema.rb                                       | 14 ++++-
 spec/controllers/api/v1/markers_controller_spec.rb | 65 ++++++++++++++++++++++
 spec/fabricators/marker_fabricator.rb              |  6 ++
 spec/models/marker_spec.rb                         |  5 ++
 12 files changed, 219 insertions(+), 2 deletions(-)
 create mode 100644 app/controllers/api/v1/markers_controller.rb
 create mode 100644 app/javascript/mastodon/actions/markers.js
 create mode 100644 app/models/marker.rb
 create mode 100644 app/serializers/rest/marker_serializer.rb
 create mode 100644 db/migrate/20190904222339_create_markers.rb
 create mode 100644 spec/controllers/api/v1/markers_controller_spec.rb
 create mode 100644 spec/fabricators/marker_fabricator.rb
 create mode 100644 spec/models/marker_spec.rb

(limited to 'spec')

diff --git a/app/controllers/api/v1/markers_controller.rb b/app/controllers/api/v1/markers_controller.rb
new file mode 100644
index 000000000..28c2ec791
--- /dev/null
+++ b/app/controllers/api/v1/markers_controller.rb
@@ -0,0 +1,44 @@
+# frozen_string_literal: true
+
+class Api::V1::MarkersController < Api::BaseController
+  before_action -> { doorkeeper_authorize! :read, :'read:statuses' }, only: [:index]
+  before_action -> { doorkeeper_authorize! :write, :'write:statuses' }, except: [:index]
+
+  before_action :require_user!
+
+  def index
+    @markers = current_user.markers.where(timeline: Array(params[:timeline])).each_with_object({}) { |marker, h| h[marker.timeline] = marker }
+    render json: serialize_map(@markers)
+  end
+
+  def create
+    Marker.transaction do
+      @markers = {}
+
+      resource_params.each_pair do |timeline, timeline_params|
+        @markers[timeline] = current_user.markers.find_or_initialize_by(timeline: timeline)
+        @markers[timeline].update!(timeline_params)
+      end
+    end
+
+    render json: serialize_map(@markers)
+  rescue ActiveRecord::StaleObjectError
+    render json: { error: 'Conflict during update, please try again' }, status: 409
+  end
+
+  private
+
+  def serialize_map(map)
+    serialized = {}
+
+    map.each_pair do |key, value|
+      serialized[key] = ActiveModelSerializers::SerializableResource.new(value, serializer: REST::MarkerSerializer).as_json
+    end
+
+    Oj.dump(serialized)
+  end
+
+  def resource_params
+    params.slice(*Marker::TIMELINES).permit(*Marker::TIMELINES.map { |timeline| { timeline.to_sym => [:last_read_id] } })
+  end
+end
diff --git a/app/javascript/mastodon/actions/markers.js b/app/javascript/mastodon/actions/markers.js
new file mode 100644
index 000000000..c3a5fe86f
--- /dev/null
+++ b/app/javascript/mastodon/actions/markers.js
@@ -0,0 +1,30 @@
+export const submitMarkers = () => (dispatch, getState) => {
+  const accessToken = getState().getIn(['meta', 'access_token'], '');
+  const params      = {};
+
+  const lastHomeId         = getState().getIn(['timelines', 'home', 'items', 0]);
+  const lastNotificationId = getState().getIn(['notifications', 'items', 0, 'id']);
+
+  if (lastHomeId) {
+    params.home = {
+      last_read_id: lastHomeId,
+    };
+  }
+
+  if (lastNotificationId) {
+    params.notifications = {
+      last_read_id: lastNotificationId,
+    };
+  }
+
+  if (Object.keys(params).length === 0) {
+    return;
+  }
+
+  const client = new XMLHttpRequest();
+
+  client.open('POST', '/api/v1/markers', false);
+  client.setRequestHeader('Content-Type', 'application/json');
+  client.setRequestHeader('Authorization', `Bearer ${accessToken}`);
+  client.send(JSON.stringify(params));
+};
diff --git a/app/javascript/mastodon/features/ui/index.js b/app/javascript/mastodon/features/ui/index.js
index 49c5c8d0e..63c5622b6 100644
--- a/app/javascript/mastodon/features/ui/index.js
+++ b/app/javascript/mastodon/features/ui/index.js
@@ -16,6 +16,7 @@ import { expandNotifications } from '../../actions/notifications';
 import { fetchFilters } from '../../actions/filters';
 import { clearHeight } from '../../actions/height_cache';
 import { focusApp, unfocusApp } from 'mastodon/actions/app';
+import { submitMarkers } from 'mastodon/actions/markers';
 import { WrappedSwitch, WrappedRoute } from './util/react_router_helpers';
 import UploadArea from './components/upload_area';
 import ColumnsAreaContainer from './containers/columns_area_container';
@@ -241,7 +242,9 @@ class UI extends React.PureComponent {
   };
 
   handleBeforeUnload = e => {
-    const { intl, isComposing, hasComposingText, hasMediaAttachments } = this.props;
+    const { intl, dispatch, isComposing, hasComposingText, hasMediaAttachments } = this.props;
+
+    dispatch(submitMarkers());
 
     if (isComposing && (hasComposingText || hasMediaAttachments)) {
       // Setting returnValue to any string causes confirmation dialog.
diff --git a/app/models/marker.rb b/app/models/marker.rb
new file mode 100644
index 000000000..a5bd2176a
--- /dev/null
+++ b/app/models/marker.rb
@@ -0,0 +1,23 @@
+# frozen_string_literal: true
+
+# == Schema Information
+#
+# Table name: markers
+#
+#  id           :bigint(8)        not null, primary key
+#  user_id      :bigint(8)
+#  timeline     :string           default(""), not null
+#  last_read_id :bigint(8)        default(0), not null
+#  lock_version :integer          default(0), not null
+#  created_at   :datetime         not null
+#  updated_at   :datetime         not null
+#
+
+class Marker < ApplicationRecord
+  TIMELINES = %w(home notifications).freeze
+
+  belongs_to :user
+
+  validates :timeline, :last_read_id, presence: true
+  validates :timeline, inclusion: { in: TIMELINES }
+end
diff --git a/app/models/user.rb b/app/models/user.rb
index a4a20d975..95f1d8fc5 100644
--- a/app/models/user.rb
+++ b/app/models/user.rb
@@ -74,6 +74,7 @@ class User < ApplicationRecord
   has_many :applications, class_name: 'Doorkeeper::Application', as: :owner
   has_many :backups, inverse_of: :user
   has_many :invites, inverse_of: :user
+  has_many :markers, inverse_of: :user, dependent: :destroy
 
   has_one :invite_request, class_name: 'UserInviteRequest', inverse_of: :user, dependent: :destroy
   accepts_nested_attributes_for :invite_request, reject_if: ->(attributes) { attributes['text'].blank? }
diff --git a/app/serializers/rest/marker_serializer.rb b/app/serializers/rest/marker_serializer.rb
new file mode 100644
index 000000000..2eaf3d507
--- /dev/null
+++ b/app/serializers/rest/marker_serializer.rb
@@ -0,0 +1,13 @@
+# frozen_string_literal: true
+
+class REST::MarkerSerializer < ActiveModel::Serializer
+  attributes :last_read_id, :version, :updated_at
+
+  def last_read_id
+    object.last_read_id.to_s
+  end
+
+  def version
+    object.lock_version
+  end
+end
diff --git a/config/routes.rb b/config/routes.rb
index c5326052e..74a162f32 100644
--- a/config/routes.rb
+++ b/config/routes.rb
@@ -314,6 +314,7 @@ Rails.application.routes.draw do
       resources :trends,       only: [:index]
       resources :filters,      only: [:index, :create, :show, :update, :destroy]
       resources :endorsements, only: [:index]
+      resources :markers,      only: [:index, :create]
 
       namespace :apps do
         get :verify_credentials, to: 'credentials#show'
diff --git a/db/migrate/20190904222339_create_markers.rb b/db/migrate/20190904222339_create_markers.rb
new file mode 100644
index 000000000..71ca70ac3
--- /dev/null
+++ b/db/migrate/20190904222339_create_markers.rb
@@ -0,0 +1,14 @@
+class CreateMarkers < ActiveRecord::Migration[5.2]
+  def change
+    create_table :markers do |t|
+      t.references :user, foreign_key: { on_delete: :cascade, index: false }
+      t.string :timeline, default: '', null: false
+      t.bigint :last_read_id, default: 0, null: false
+      t.integer :lock_version, default: 0, null: false
+
+      t.timestamps
+    end
+
+    add_index :markers, [:user_id, :timeline], unique: true
+  end
+end
diff --git a/db/schema.rb b/db/schema.rb
index 5576f70bf..834dddd7b 100644
--- a/db/schema.rb
+++ b/db/schema.rb
@@ -10,7 +10,7 @@
 #
 # It's strongly recommended that you check this file into your version control system.
 
-ActiveRecord::Schema.define(version: 2019_09_01_040524) do
+ActiveRecord::Schema.define(version: 2019_09_04_222339) do
 
   # These are extensions that must be enabled in order to support this database
   enable_extension "plpgsql"
@@ -366,6 +366,17 @@ ActiveRecord::Schema.define(version: 2019_09_01_040524) do
     t.index ["account_id"], name: "index_lists_on_account_id"
   end
 
+  create_table "markers", force: :cascade do |t|
+    t.bigint "user_id"
+    t.string "timeline", default: "", null: false
+    t.bigint "last_read_id", default: 0, null: false
+    t.integer "lock_version", default: 0, null: false
+    t.datetime "created_at", null: false
+    t.datetime "updated_at", null: false
+    t.index ["user_id", "timeline"], name: "index_markers_on_user_id_and_timeline", unique: true
+    t.index ["user_id"], name: "index_markers_on_user_id"
+  end
+
   create_table "media_attachments", force: :cascade do |t|
     t.bigint "status_id"
     t.string "file_file_name"
@@ -791,6 +802,7 @@ ActiveRecord::Schema.define(version: 2019_09_01_040524) do
   add_foreign_key "list_accounts", "follows", on_delete: :cascade
   add_foreign_key "list_accounts", "lists", on_delete: :cascade
   add_foreign_key "lists", "accounts", on_delete: :cascade
+  add_foreign_key "markers", "users", on_delete: :cascade
   add_foreign_key "media_attachments", "accounts", name: "fk_96dd81e81b", on_delete: :nullify
   add_foreign_key "media_attachments", "scheduled_statuses", on_delete: :nullify
   add_foreign_key "media_attachments", "statuses", on_delete: :nullify
diff --git a/spec/controllers/api/v1/markers_controller_spec.rb b/spec/controllers/api/v1/markers_controller_spec.rb
new file mode 100644
index 000000000..556a75b9b
--- /dev/null
+++ b/spec/controllers/api/v1/markers_controller_spec.rb
@@ -0,0 +1,65 @@
+require 'rails_helper'
+
+RSpec.describe Api::V1::MarkersController, type: :controller do
+  render_views
+
+  let!(:user)  { Fabricate(:user, account: Fabricate(:account, username: 'alice')) }
+  let!(:token) { Fabricate(:accessible_access_token, resource_owner_id: user.id, scopes: 'read:statuses write:statuses') }
+
+  before { allow(controller).to receive(:doorkeeper_token) { token } }
+
+  describe 'GET #index' do
+    before do
+      Fabricate(:marker, timeline: 'home', last_read_id: 123, user: user)
+      Fabricate(:marker, timeline: 'notifications', last_read_id: 456, user: user)
+
+      get :index, params: { timeline: %w(home notifications) }
+    end
+
+    it 'returns http success' do
+      expect(response).to have_http_status(200)
+    end
+
+    it 'returns markers' do
+      json = body_as_json
+
+      expect(json.key?(:home)).to be true
+      expect(json[:home][:last_read_id]).to eq '123'
+      expect(json.key?(:notifications)).to be true
+      expect(json[:notifications][:last_read_id]).to eq '456'
+    end
+  end
+
+  describe 'POST #create' do
+    context 'when no marker exists' do
+      before do
+        post :create, params: { home: { last_read_id: '69420' } }
+      end
+
+      it 'returns http success' do
+        expect(response).to have_http_status(200)
+      end
+
+      it 'creates a marker' do
+        expect(user.markers.first.timeline).to eq 'home'
+        expect(user.markers.first.last_read_id).to eq 69420
+      end
+    end
+
+    context 'when a marker exists' do
+      before do
+        post :create, params: { home: { last_read_id: '69420' } }
+        post :create, params: { home: { last_read_id: '70120' } }
+      end
+
+      it 'returns http success' do
+        expect(response).to have_http_status(200)
+      end
+
+      it 'updates a marker' do
+        expect(user.markers.first.timeline).to eq 'home'
+        expect(user.markers.first.last_read_id).to eq 70120
+      end
+    end
+  end
+end
diff --git a/spec/fabricators/marker_fabricator.rb b/spec/fabricators/marker_fabricator.rb
new file mode 100644
index 000000000..0c94150e0
--- /dev/null
+++ b/spec/fabricators/marker_fabricator.rb
@@ -0,0 +1,6 @@
+Fabricator(:marker) do
+  user
+  timeline     'home'
+  last_read_id 0
+  lock_version 0
+end
diff --git a/spec/models/marker_spec.rb b/spec/models/marker_spec.rb
new file mode 100644
index 000000000..d716aa75c
--- /dev/null
+++ b/spec/models/marker_spec.rb
@@ -0,0 +1,5 @@
+require 'rails_helper'
+
+RSpec.describe Marker, type: :model do
+  pending "add some examples to (or delete) #{__FILE__}"
+end
-- 
cgit 


From 1110ea1a9162d5488e1ed5dbccd0803618e713f8 Mon Sep 17 00:00:00 2001
From: Eugen Rochko <eugen@zeonfederated.com>
Date: Mon, 9 Sep 2019 22:44:17 +0200
Subject: Add batch actions and categories to admin UI for custom emojis
 (#11793)

---
 app/controllers/admin/custom_emojis_controller.rb  | 102 +++++++-------------
 app/javascript/styles/mastodon/tables.scss         |  41 ++++++++
 app/models/custom_emoji.rb                         |   6 ++
 app/models/custom_emoji_category.rb                |   2 +
 app/models/custom_emoji_filter.rb                  |   8 +-
 app/models/form/custom_emoji_batch.rb              | 106 +++++++++++++++++++++
 .../admin/custom_emojis/_custom_emoji.html.haml    |  55 ++++++-----
 app/views/admin/custom_emojis/index.html.haml      |  66 ++++++++++---
 config/locales/en.yml                              |   3 +
 config/routes.rb                                   |   8 +-
 .../admin/custom_emojis_controller_spec.rb         |  60 ------------
 11 files changed, 281 insertions(+), 176 deletions(-)
 create mode 100644 app/models/form/custom_emoji_batch.rb

(limited to 'spec')

diff --git a/app/controllers/admin/custom_emojis_controller.rb b/app/controllers/admin/custom_emojis_controller.rb
index f77699166..2af90f051 100644
--- a/app/controllers/admin/custom_emojis_controller.rb
+++ b/app/controllers/admin/custom_emojis_controller.rb
@@ -2,19 +2,20 @@
 
 module Admin
   class CustomEmojisController < BaseController
-    before_action :set_custom_emoji, except: [:index, :new, :create]
-    before_action :set_filter_params
-
     include ObfuscateFilename
+
     obfuscate_filename [:custom_emoji, :image]
 
     def index
       authorize :custom_emoji, :index?
+
       @custom_emojis = filtered_custom_emojis.eager_load(:local_counterpart).page(params[:page])
+      @form          = Form::CustomEmojiBatch.new
     end
 
     def new
       authorize :custom_emoji, :create?
+
       @custom_emoji = CustomEmoji.new
     end
 
@@ -31,69 +32,17 @@ module Admin
       end
     end
 
-    def update
-      authorize @custom_emoji, :update?
-
-      if @custom_emoji.update(resource_params)
-        log_action :update, @custom_emoji
-        flash[:notice] = I18n.t('admin.custom_emojis.updated_msg')
-      else
-        flash[:alert] =  I18n.t('admin.custom_emojis.update_failed_msg')
-      end
-      redirect_to admin_custom_emojis_path(page: params[:page], **@filter_params)
-    end
-
-    def destroy
-      authorize @custom_emoji, :destroy?
-      @custom_emoji.destroy!
-      log_action :destroy, @custom_emoji
-      flash[:notice] = I18n.t('admin.custom_emojis.destroyed_msg')
-      redirect_to admin_custom_emojis_path(page: params[:page], **@filter_params)
-    end
-
-    def copy
-      authorize @custom_emoji, :copy?
-
-      emoji = CustomEmoji.find_or_initialize_by(domain: nil,
-                                                shortcode: @custom_emoji.shortcode)
-      emoji.image = @custom_emoji.image
-
-      if emoji.save
-        log_action :create, emoji
-        flash[:notice] = I18n.t('admin.custom_emojis.copied_msg')
-      else
-        flash[:alert] = I18n.t('admin.custom_emojis.copy_failed_msg')
-      end
-
-      redirect_to admin_custom_emojis_path(page: params[:page], **@filter_params)
-    end
-
-    def enable
-      authorize @custom_emoji, :enable?
-      @custom_emoji.update!(disabled: false)
-      log_action :enable, @custom_emoji
-      flash[:notice] = I18n.t('admin.custom_emojis.enabled_msg')
-      redirect_to admin_custom_emojis_path(page: params[:page], **@filter_params)
-    end
-
-    def disable
-      authorize @custom_emoji, :disable?
-      @custom_emoji.update!(disabled: true)
-      log_action :disable, @custom_emoji
-      flash[:notice] = I18n.t('admin.custom_emojis.disabled_msg')
-      redirect_to admin_custom_emojis_path(page: params[:page], **@filter_params)
+    def batch
+      @form = Form::CustomEmojiBatch.new(form_custom_emoji_batch_params.merge(current_account: current_account, action: action_from_button))
+      @form.save
+    rescue ActionController::ParameterMissing
+      flash[:alert] = I18n.t('admin.accounts.no_account_selected')
+    ensure
+      redirect_to admin_custom_emojis_path(filter_params)
     end
 
     private
 
-    def set_custom_emoji
-      @custom_emoji = CustomEmoji.find(params[:id])
-    end
-
-    def set_filter_params
-      @filter_params = filter_params.to_hash.symbolize_keys
-    end
-
     def resource_params
       params.require(:custom_emoji).permit(:shortcode, :image, :visible_in_picker)
     end
@@ -103,12 +52,29 @@ module Admin
     end
 
     def filter_params
-      params.permit(
-        :local,
-        :remote,
-        :by_domain,
-        :shortcode
-      )
+      params.slice(:local, :remote, :by_domain, :shortcode, :page).permit(:local, :remote, :by_domain, :shortcode, :page)
+    end
+
+    def action_from_button
+      if params[:update]
+        'update'
+      elsif params[:list]
+        'list'
+      elsif params[:unlist]
+        'unlist'
+      elsif params[:enable]
+        'enable'
+      elsif params[:disable]
+        'disable'
+      elsif params[:copy]
+        'copy'
+      elsif params[:delete]
+        'delete'
+      end
+    end
+
+    def form_custom_emoji_batch_params
+      params.require(:form_custom_emoji_batch).permit(:action, :category_id, :category_name, custom_emoji_ids: [])
     end
   end
 end
diff --git a/app/javascript/styles/mastodon/tables.scss b/app/javascript/styles/mastodon/tables.scss
index 2aef099e6..d6403986f 100644
--- a/app/javascript/styles/mastodon/tables.scss
+++ b/app/javascript/styles/mastodon/tables.scss
@@ -180,6 +180,18 @@ a.table-action-link {
     }
   }
 
+  &__form {
+    padding: 16px;
+    border: 1px solid darken($ui-base-color, 8%);
+    border-top: 0;
+    background: $ui-base-color;
+
+    .fields-row {
+      padding-top: 0;
+      margin-bottom: 0;
+    }
+  }
+
   &__row {
     border: 1px solid darken($ui-base-color, 8%);
     border-top: 0;
@@ -210,6 +222,35 @@ a.table-action-link {
       &--unpadded {
         padding: 0;
       }
+
+      &--with-image {
+        display: flex;
+        align-items: center;
+      }
+
+      &__image {
+        flex: 0 0 auto;
+        display: flex;
+        justify-content: center;
+        align-items: center;
+        margin-right: 10px;
+
+        .emojione {
+          width: 32px;
+          height: 32px;
+        }
+      }
+
+      &__text {
+        flex: 1 1 auto;
+      }
+
+      &__extra {
+        flex: 0 0 auto;
+        text-align: right;
+        color: $darker-text-color;
+        font-weight: 500;
+      }
     }
 
     .directory__tag {
diff --git a/app/models/custom_emoji.rb b/app/models/custom_emoji.rb
index b21ad9042..0a4201a14 100644
--- a/app/models/custom_emoji.rb
+++ b/app/models/custom_emoji.rb
@@ -59,6 +59,12 @@ class CustomEmoji < ApplicationRecord
     :emoji
   end
 
+  def copy!
+    copy = self.class.find_or_initialize_by(domain: nil, shortcode: shortcode)
+    copy.image = image
+    copy.save!
+  end
+
   class << self
     def from_text(text, domain)
       return [] if text.blank?
diff --git a/app/models/custom_emoji_category.rb b/app/models/custom_emoji_category.rb
index 7d8c0ee2d..3c87f2b2e 100644
--- a/app/models/custom_emoji_category.rb
+++ b/app/models/custom_emoji_category.rb
@@ -12,4 +12,6 @@
 
 class CustomEmojiCategory < ApplicationRecord
   has_many :emojis, class_name: 'CustomEmoji', foreign_key: 'category_id', inverse_of: :category
+
+  validates :name, presence: true, uniqueness: true
 end
diff --git a/app/models/custom_emoji_filter.rb b/app/models/custom_emoji_filter.rb
index 7649055d2..15b8da1d1 100644
--- a/app/models/custom_emoji_filter.rb
+++ b/app/models/custom_emoji_filter.rb
@@ -11,6 +11,8 @@ class CustomEmojiFilter
     scope = CustomEmoji.alphabetic
 
     params.each do |key, value|
+      next if key.to_s == 'page'
+
       scope.merge!(scope_for(key, value)) if value.present?
     end
 
@@ -22,13 +24,13 @@ class CustomEmojiFilter
   def scope_for(key, value)
     case key.to_s
     when 'local'
-      CustomEmoji.local
+      CustomEmoji.local.left_joins(:category).reorder(Arel.sql('custom_emoji_categories.name ASC NULLS FIRST, custom_emojis.shortcode ASC'))
     when 'remote'
       CustomEmoji.remote
     when 'by_domain'
-      CustomEmoji.where(domain: value.downcase)
+      CustomEmoji.where(domain: value.strip.downcase)
     when 'shortcode'
-      CustomEmoji.search(value)
+      CustomEmoji.search(value.strip)
     else
       raise "Unknown filter: #{key}"
     end
diff --git a/app/models/form/custom_emoji_batch.rb b/app/models/form/custom_emoji_batch.rb
new file mode 100644
index 000000000..076e8c9e3
--- /dev/null
+++ b/app/models/form/custom_emoji_batch.rb
@@ -0,0 +1,106 @@
+# frozen_string_literal: true
+
+class Form::CustomEmojiBatch
+  include ActiveModel::Model
+  include Authorization
+  include AccountableConcern
+
+  attr_accessor :custom_emoji_ids, :action, :current_account,
+                :category_id, :category_name, :visible_in_picker
+
+  def save
+    case action
+    when 'update'
+      update!
+    when 'list'
+      list!
+    when 'unlist'
+      unlist!
+    when 'enable'
+      enable!
+    when 'disable'
+      disable!
+    when 'copy'
+      copy!
+    when 'delete'
+      delete!
+    end
+  end
+
+  private
+
+  def custom_emojis
+    CustomEmoji.where(id: custom_emoji_ids)
+  end
+
+  def update!
+    custom_emojis.each { |custom_emoji| authorize(custom_emoji, :update?) }
+
+    category = begin
+      if category_id.present?
+        CustomEmojiCategory.find(category_id)
+      elsif category_name.present?
+        CustomEmojiCategory.create!(name: category_name)
+      end
+    end
+
+    custom_emojis.each do |custom_emoji|
+      custom_emoji.update(category_id: category&.id)
+      log_action :update, custom_emoji
+    end
+  end
+
+  def list!
+    custom_emojis.each { |custom_emoji| authorize(custom_emoji, :update?) }
+
+    custom_emojis.each do |custom_emoji|
+      custom_emoji.update(visible_in_picker: true)
+      log_action :update, custom_emoji
+    end
+  end
+
+  def unlist!
+    custom_emojis.each { |custom_emoji| authorize(custom_emoji, :update?) }
+
+    custom_emojis.each do |custom_emoji|
+      custom_emoji.update(visible_in_picker: false)
+      log_action :update, custom_emoji
+    end
+  end
+
+  def enable!
+    custom_emojis.each { |custom_emoji| authorize(custom_emoji, :enable?) }
+
+    custom_emojis.each do |custom_emoji|
+      custom_emoji.update(disabled: false)
+      log_action :enable, custom_emoji
+    end
+  end
+
+  def disable!
+    custom_emojis.each { |custom_emoji| authorize(custom_emoji, :disable?) }
+
+    custom_emojis.each do |custom_emoji|
+      custom_emoji.update(disabled: true)
+      log_action :disable, custom_emoji
+    end
+  end
+
+  def copy!
+    custom_emojis.each { |custom_emoji| authorize(custom_emoji, :copy?) }
+
+    custom_emojis.each do |custom_emoji|
+      copied_custom_emoji = custom_emoji.copy!
+      log_action :create, copied_custom_emoji
+    end
+  end
+
+  def delete!
+    custom_emojis.each { |custom_emoji| authorize(custom_emoji, :destroy?) }
+
+    custom_emojis.each do |custom_emoji|
+      custom_emoji.destroy
+      log_action :destroy, custom_emoji
+    end
+  end
+end
diff --git a/app/views/admin/custom_emojis/_custom_emoji.html.haml b/app/views/admin/custom_emojis/_custom_emoji.html.haml
index fbaa9a174..9e06a3b42 100644
--- a/app/views/admin/custom_emojis/_custom_emoji.html.haml
+++ b/app/views/admin/custom_emojis/_custom_emoji.html.haml
@@ -1,28 +1,31 @@
-%tr
-  %td
-    = custom_emoji_tag(custom_emoji)
-  %td
-    %samp= ":#{custom_emoji.shortcode}:"
-  %td
-    - if custom_emoji.local?
-      = t('admin.accounts.location.local')
-    - else
-      = link_to custom_emoji.domain, admin_custom_emojis_path(by_domain: custom_emoji.domain)
-  %td
-    - if custom_emoji.local?
-      - if custom_emoji.visible_in_picker
-        = table_link_to 'eye', t('admin.custom_emojis.listed'), admin_custom_emoji_path(custom_emoji, custom_emoji: { visible_in_picker: false }, page: params[:page], **@filter_params), method: :patch
+.batch-table__row
+  %label.batch-table__row__select.batch-table__row__select--aligned.batch-checkbox
+    = f.check_box :custom_emoji_ids, { multiple: true, include_hidden: false }, custom_emoji.id
+  .batch-table__row__content.batch-table__row__content--with-image
+    .batch-table__row__content__image
+      = custom_emoji_tag(custom_emoji)
+
+    .batch-table__row__content__text
+      %samp= ":#{custom_emoji.shortcode}:"
+
+      - if custom_emoji.local?
+        %span.account-role.bot= custom_emoji.category&.name || t('admin.custom_emojis.uncategorized')
+
+    .batch-table__row__content__extra
+      - if custom_emoji.local?
+        = t('admin.accounts.location.local')
       - else
-        = table_link_to 'eye-slash', t('admin.custom_emojis.unlisted'), admin_custom_emoji_path(custom_emoji, custom_emoji: { visible_in_picker: true }, page: params[:page], **@filter_params), method: :patch
-    - else
-      - if custom_emoji.local_counterpart.present?
-        = link_to safe_join([custom_emoji_tag(custom_emoji.local_counterpart), t('admin.custom_emojis.overwrite')]), copy_admin_custom_emoji_path(custom_emoji, page: params[:page], **@filter_params), method: :post, class: 'table-action-link'
+        = custom_emoji.domain
+
+      %br/
+
+      - if custom_emoji.disabled?
+        = t('admin.custom_emojis.disabled')
       - else
-        = table_link_to 'copy', t('admin.custom_emojis.copy'), copy_admin_custom_emoji_path(custom_emoji, page: params[:page], **@filter_params), method: :post
-  %td
-    - if custom_emoji.disabled?
-      = table_link_to 'power-off', t('admin.custom_emojis.enable'), enable_admin_custom_emoji_path(custom_emoji, page: params[:page], **@filter_params), method: :post, data: { confirm: t('admin.accounts.are_you_sure') }
-    - else
-      = table_link_to 'power-off', t('admin.custom_emojis.disable'), disable_admin_custom_emoji_path(custom_emoji, page: params[:page], **@filter_params), method: :post, data: { confirm: t('admin.accounts.are_you_sure') }
-  %td
-    = table_link_to 'times', t('admin.custom_emojis.delete'), admin_custom_emoji_path(custom_emoji, page: params[:page], **@filter_params), method: :delete, data: { confirm: t('admin.accounts.are_you_sure') }
+        = t('admin.custom_emojis.enabled')
+      - if custom_emoji.local?
+        &bull;
+        - if custom_emoji.visible_in_picker?
+          = t('admin.custom_emojis.listed')
+        - else
+          = t('admin.custom_emojis.unlisted')
diff --git a/app/views/admin/custom_emojis/index.html.haml b/app/views/admin/custom_emojis/index.html.haml
index 3a119276c..7320ce1bb 100644
--- a/app/views/admin/custom_emojis/index.html.haml
+++ b/app/views/admin/custom_emojis/index.html.haml
@@ -1,6 +1,9 @@
 - content_for :page_title do
   = t('admin.custom_emojis.title')
 
+- content_for :header_tags do
+  = javascript_pack_tag 'admin', integrity: true, async: true, crossorigin: 'anonymous'
+
 .filters
   .filter-subset
     %strong= t('admin.accounts.location.title')
@@ -20,8 +23,7 @@
 = form_tag admin_custom_emojis_url, method: 'GET', class: 'simple_form' do
   .fields-group
     - Admin::FilterHelper::CUSTOM_EMOJI_FILTERS.each do |key|
-      - if params[key].present?
-        = hidden_field_tag key, params[key]
+      = hidden_field_tag key, params[key] if params[key].present?
 
     - %i(shortcode by_domain).each do |key|
       .input.string.optional
@@ -31,18 +33,54 @@
       %button= t('admin.accounts.search')
       = link_to t('admin.accounts.reset'), admin_custom_emojis_path, class: 'button negative'
 
-.table-wrapper
-  %table.table
-    %thead
-      %tr
-        %th= t('admin.custom_emojis.emoji')
-        %th= t('admin.custom_emojis.shortcode')
-        %th= t('admin.accounts.domain')
-        %th
-        %th
-        %th
-    %tbody
-      = render @custom_emojis
+= form_for(@form, url: batch_admin_custom_emojis_path) do |f|
+  = hidden_field_tag :page, params[:page] || 1
+
+  - Admin::FilterHelper::CUSTOM_EMOJI_FILTERS.each do |key|
+    = hidden_field_tag key, params[key] if params[key].present?
+
+  .batch-table
+    .batch-table__toolbar
+      %label.batch-table__toolbar__select.batch-checkbox-all
+        = check_box_tag :batch_checkbox_all, nil, false
+      .batch-table__toolbar__actions
+        - if params[:local] == '1'
+          = f.button safe_join([fa_icon('save'), t('generic.save_changes')]), name: :update, class: 'table-action-link', type: :submit, data: { confirm: t('admin.reports.are_you_sure') }
+
+          = f.button safe_join([fa_icon('eye'), t('admin.custom_emojis.list')]), name: :list, class: 'table-action-link', type: :submit, data: { confirm: t('admin.reports.are_you_sure') }
+
+          = f.button safe_join([fa_icon('eye-slash'), t('admin.custom_emojis.unlist')]), name: :unlist, class: 'table-action-link', type: :submit, data: { confirm: t('admin.reports.are_you_sure') }
+
+        = f.button safe_join([fa_icon('power-off'), t('admin.custom_emojis.enable')]), name: :enable, class: 'table-action-link', type: :submit, data: { confirm: t('admin.reports.are_you_sure') }
+
+        = f.button safe_join([fa_icon('power-off'), t('admin.custom_emojis.disable')]), name: :disable, class: 'table-action-link', type: :submit, data: { confirm: t('admin.reports.are_you_sure') }
+
+        = f.button safe_join([fa_icon('times'), t('admin.custom_emojis.delete')]), name: :delete, class: 'table-action-link', type: :submit, data: { confirm: t('admin.reports.are_you_sure') }
+
+        - unless params[:local] == '1'
+          = f.button safe_join([fa_icon('copy'), t('admin.custom_emojis.copy')]), name: :copy, class: 'table-action-link', type: :submit, data: { confirm: t('admin.reports.are_you_sure') }
+
+    - if params[:local] == '1'
+      .batch-table__form.simple_form
+        .fields-row
+          .fields-group.fields-row__column.fields-row__column-6
+            .input.select.optional
+              .label_input
+                = f.select :category_id, options_from_collection_for_select(CustomEmojiCategory.all, 'id', 'name'), prompt: t('admin.custom_emojis.assign_category'), class: 'select optional', 'aria-label': t('admin.custom_emojis.assign_category')
+
+          .fields-group.fields-row__column.fields-row__column-6
+            .input.string.optional
+              .label_input
+                = f.text_field :category_name, class: 'string optional', placeholder: t('admin.custom_emojis.create_new_category'), 'aria-label': t('admin.custom_emojis.create_new_category')
+
+    .batch-table__body
+      - if @custom_emojis.empty?
+        = nothing_here 'nothing-here--under-tabs'
+      - else
+        = render partial: 'custom_emoji', collection: @custom_emojis, locals: { f: f }
 
 = paginate @custom_emojis
+
+%hr.spacer/
+
 = link_to t('admin.custom_emojis.upload'), new_admin_custom_emoji_path, class: 'button'
diff --git a/config/locales/en.yml b/config/locales/en.yml
index 42d8e0eb8..52cb4a269 100644
--- a/config/locales/en.yml
+++ b/config/locales/en.yml
@@ -225,10 +225,12 @@ en:
       deleted_status: "(deleted status)"
       title: Audit log
     custom_emojis:
+      assign_category: Assign category
       by_domain: Domain
       copied_msg: Successfully created local copy of the emoji
       copy: Copy
       copy_failed_msg: Could not make a local copy of that emoji
+      create_new_category: Create new category
       created_msg: Emoji successfully created!
       delete: Delete
       destroyed_msg: Emojo successfully destroyed!
@@ -245,6 +247,7 @@ en:
       shortcode: Shortcode
       shortcode_hint: At least 2 characters, only alphanumeric characters and underscores
       title: Custom emojis
+      uncategorized: Uncategorized
       unlisted: Unlisted
       update_failed_msg: Could not update that emoji
       updated_msg: Emoji successfully updated!
diff --git a/config/routes.rb b/config/routes.rb
index 534e68814..d22a9e56a 100644
--- a/config/routes.rb
+++ b/config/routes.rb
@@ -242,11 +242,9 @@ Rails.application.routes.draw do
       resource :two_factor_authentication, only: [:destroy]
     end
 
-    resources :custom_emojis, only: [:index, :new, :create, :update, :destroy] do
-      member do
-        post :copy
-        post :enable
-        post :disable
+    resources :custom_emojis, only: [:index, :new, :create] do
+      collection do
+        post :batch
       end
     end
 
diff --git a/spec/controllers/admin/custom_emojis_controller_spec.rb b/spec/controllers/admin/custom_emojis_controller_spec.rb
index b7e2894e9..a8d96948c 100644
--- a/spec/controllers/admin/custom_emojis_controller_spec.rb
+++ b/spec/controllers/admin/custom_emojis_controller_spec.rb
@@ -52,64 +52,4 @@ describe Admin::CustomEmojisController do
       end
     end
   end
-
-  describe 'PUT #update' do
-    let(:custom_emoji) { Fabricate(:custom_emoji, shortcode: 'test') }
-    let(:image) { fixture_file_upload(Rails.root.join('spec', 'fixtures', 'files', 'emojo.png'), 'image/png') }
-
-    before do
-      put :update, params: { id: custom_emoji.id, custom_emoji: params }
-    end
-
-    context 'when parameter is valid' do
-      let(:params) { { shortcode: 'updated', image: image } }
-
-      it 'succeeds in updating custom emoji' do
-        expect(flash[:notice]).to eq I18n.t('admin.custom_emojis.updated_msg')
-        expect(custom_emoji.reload).to have_attributes(shortcode: 'updated')
-      end
-    end
-
-    context 'when parameter is invalid' do
-      let(:params) { { shortcode: 'u', image: image } }
-
-      it 'fails to update custom emoji' do
-        expect(flash[:alert]).to eq I18n.t('admin.custom_emojis.update_failed_msg')
-        expect(custom_emoji.reload).to have_attributes(shortcode: 'test')
-      end
-    end
-  end
-
-  describe 'POST #copy' do
-    subject { post :copy, params: { id: custom_emoji.id } }
-
-    let(:custom_emoji) { Fabricate(:custom_emoji, shortcode: 'test') }
-
-    it 'copies custom emoji' do
-      expect { subject }.to change { CustomEmoji.where(shortcode: 'test').count }.by(1)
-      expect(flash[:notice]).to eq I18n.t('admin.custom_emojis.copied_msg')
-    end
-  end
-
-  describe 'POST #enable' do
-    let(:custom_emoji) { Fabricate(:custom_emoji, shortcode: 'test', disabled: true) }
-
-    before { post :enable, params: { id: custom_emoji.id } }
-
-    it 'enables custom emoji' do
-      expect(response).to redirect_to admin_custom_emojis_path
-      expect(custom_emoji.reload).to have_attributes(disabled: false)
-    end
-  end
-
-  describe 'POST #disable' do
-    let(:custom_emoji) { Fabricate(:custom_emoji, shortcode: 'test', disabled: false) }
-
-    before { post :disable, params: { id: custom_emoji.id } }
-
-    it 'enables custom emoji' do
-      expect(response).to redirect_to admin_custom_emojis_path
-      expect(custom_emoji.reload).to have_attributes(disabled: true)
-    end
-  end
 end
-- 
cgit 


From 4faaa5b25e54f863fdf3a68edc5ca664e5a84e4c Mon Sep 17 00:00:00 2001
From: ThibG <thib@sitedethib.com>
Date: Tue, 10 Sep 2019 20:56:42 +0200
Subject: Add updated relationship to follow request API responses (#11800)

Fixes #11747
---
 app/controllers/api/v1/follow_requests_controller.rb       |  8 ++++++--
 spec/controllers/api/v1/follow_requests_controller_spec.rb | 12 ++++++++++++
 2 files changed, 18 insertions(+), 2 deletions(-)

(limited to 'spec')

diff --git a/app/controllers/api/v1/follow_requests_controller.rb b/app/controllers/api/v1/follow_requests_controller.rb
index e6888154e..0ee6e531f 100644
--- a/app/controllers/api/v1/follow_requests_controller.rb
+++ b/app/controllers/api/v1/follow_requests_controller.rb
@@ -14,12 +14,12 @@ class Api::V1::FollowRequestsController < Api::BaseController
   def authorize
     AuthorizeFollowService.new.call(account, current_account)
     NotifyService.new.call(current_account, Follow.find_by(account: account, target_account: current_account))
-    render_empty
+    render json: account, serializer: REST::RelationshipSerializer, relationships: relationships
   end
 
   def reject
     RejectFollowService.new.call(account, current_account)
-    render_empty
+    render json: account, serializer: REST::RelationshipSerializer, relationships: relationships
   end
 
   private
@@ -28,6 +28,10 @@ class Api::V1::FollowRequestsController < Api::BaseController
     Account.find(params[:id])
   end
 
+  def relationships(**options)
+    AccountRelationshipsPresenter.new([params[:id]], current_user.account_id, options)
+  end
+
   def load_accounts
     default_accounts.merge(paginated_follow_requests).to_a
   end
diff --git a/spec/controllers/api/v1/follow_requests_controller_spec.rb b/spec/controllers/api/v1/follow_requests_controller_spec.rb
index 87292d9ce..ae92a9627 100644
--- a/spec/controllers/api/v1/follow_requests_controller_spec.rb
+++ b/spec/controllers/api/v1/follow_requests_controller_spec.rb
@@ -38,6 +38,12 @@ RSpec.describe Api::V1::FollowRequestsController, type: :controller do
     it 'allows follower to follow' do
       expect(follower.following?(user.account)).to be true
     end
+
+    it 'returns JSON with followed_by=true' do
+      json = body_as_json
+
+      expect(json[:followed_by]).to be true
+    end
   end
 
   describe 'POST #reject' do
@@ -54,5 +60,11 @@ RSpec.describe Api::V1::FollowRequestsController, type: :controller do
     it 'removes follow request' do
       expect(FollowRequest.where(target_account: user.account, account: follower).count).to eq 0
     end
+
+    it 'returns JSON with followed_by=false' do
+      json = body_as_json
+
+      expect(json[:followed_by]).to be false
+    end
   end
 end
-- 
cgit 


From 4fe127664b0ae22a528b4a4467ab2de92e3da3ef Mon Sep 17 00:00:00 2001
From: Tao Bror Bojlén <brortao@users.noreply.github.com>
Date: Wed, 11 Sep 2019 07:44:58 +0100
Subject: add admin setting for default search engine indexing (fix #11750)
 (#11804)

---
 app/lib/settings/scoped_settings.rb             | 1 +
 app/models/form/admin_settings.rb               | 2 ++
 app/views/admin/settings/edit.html.haml         | 3 +++
 config/locales/en.yml                           | 3 +++
 spec/controllers/application_controller_spec.rb | 1 +
 5 files changed, 10 insertions(+)

(limited to 'spec')

diff --git a/app/lib/settings/scoped_settings.rb b/app/lib/settings/scoped_settings.rb
index 3653ab114..9ca39510a 100644
--- a/app/lib/settings/scoped_settings.rb
+++ b/app/lib/settings/scoped_settings.rb
@@ -4,6 +4,7 @@ module Settings
   class ScopedSettings
     DEFAULTING_TO_UNSCOPED = %w(
       theme
+      noindex
     ).freeze
 
     def initialize(object)
diff --git a/app/models/form/admin_settings.rb b/app/models/form/admin_settings.rb
index 6bc3ca9f5..24196e182 100644
--- a/app/models/form/admin_settings.rb
+++ b/app/models/form/admin_settings.rb
@@ -32,6 +32,7 @@ class Form::AdminSettings
     trends
     show_domain_blocks
     show_domain_blocks_rationale
+    noindex
   ).freeze
 
   BOOLEAN_KEYS = %i(
@@ -45,6 +46,7 @@ class Form::AdminSettings
     profile_directory
     spam_check_enabled
     trends
+    noindex
   ).freeze
 
   UPLOAD_KEYS = %i(
diff --git a/app/views/admin/settings/edit.html.haml b/app/views/admin/settings/edit.html.haml
index 28880c087..752386b3c 100644
--- a/app/views/admin/settings/edit.html.haml
+++ b/app/views/admin/settings/edit.html.haml
@@ -71,6 +71,9 @@
     .fields-group
       = f.input :trends, as: :boolean, wrapper: :with_label, label: t('admin.settings.trends.title'), hint: t('admin.settings.trends.desc_html')
 
+    .fields-group
+      = f.input :noindex, as: :boolean, wrapper: :with_label, label: t('admin.settings.default_noindex.title'), hint: t('admin.settings.default_noindex.desc_html')
+
   .fields-group
     = f.input :spam_check_enabled, as: :boolean, wrapper: :with_label, label: t('admin.settings.spam_check_enabled.title'), hint: t('admin.settings.spam_check_enabled.desc_html')
 
diff --git a/config/locales/en.yml b/config/locales/en.yml
index 52cb4a269..0a5ca31c1 100644
--- a/config/locales/en.yml
+++ b/config/locales/en.yml
@@ -427,6 +427,9 @@ en:
       custom_css:
         desc_html: Modify the look with CSS loaded on every page
         title: Custom CSS
+      default_noindex:
+        desc_html: Affects all users who have not changed this setting themselves
+        title: Opt users out of search engine indexing by default
       domain_blocks:
         all: To everyone
         disabled: To no one
diff --git a/spec/controllers/application_controller_spec.rb b/spec/controllers/application_controller_spec.rb
index 1811500df..da4a794cd 100644
--- a/spec/controllers/application_controller_spec.rb
+++ b/spec/controllers/application_controller_spec.rb
@@ -110,6 +110,7 @@ describe ApplicationController, type: :controller do
       sign_in current_user
 
       allow(Setting).to receive(:[]).with('theme').and_return 'contrast'
+      allow(Setting).to receive(:[]).with('noindex').and_return false
 
       expect(controller.view_context.current_theme).to eq 'contrast'
     end
-- 
cgit 


From c5d37f18cb3f4d6212fb8f3e1c4e1e027f677ec5 Mon Sep 17 00:00:00 2001
From: Eugen Rochko <eugen@zeonfederated.com>
Date: Wed, 11 Sep 2019 16:32:44 +0200
Subject: Change deletes to preserve soft-deleted statuses in unresolved
 reports (#11805)

Change all account actions except "none" to resolve all unresolved reports

Refactor `SuspendAccountService` to be more readable
---
 app/controllers/admin/accounts_controller.rb       |  2 +-
 app/controllers/admin/report_notes_controller.rb   |  9 ++--
 .../api/v1/admin/accounts_controller.rb            |  2 +-
 app/lib/activitypub/activity/delete.rb             |  3 +-
 app/models/account.rb                              |  1 +
 app/models/admin/account_action.rb                 | 24 +++++++--
 app/models/form/account_batch.rb                   |  2 +-
 app/models/form/status_batch.rb                    |  2 +-
 app/models/report.rb                               |  1 +
 app/models/status.rb                               |  4 ++
 app/models/user.rb                                 |  4 ++
 app/services/block_domain_service.rb               |  2 +-
 app/services/remove_status_service.rb              |  7 +--
 app/services/suspend_account_service.rb            | 62 ++++++++++++++++------
 app/services/unallow_domain_service.rb             |  2 +-
 app/workers/admin/suspension_worker.rb             |  2 +-
 lib/mastodon/accounts_cli.rb                       |  4 +-
 lib/mastodon/domains_cli.rb                        |  2 +-
 .../admin/reported_statuses_controller_spec.rb     |  2 +-
 spec/controllers/admin/statuses_controller_spec.rb |  2 +-
 spec/models/form/status_batch_spec.rb              |  4 +-
 21 files changed, 98 insertions(+), 45 deletions(-)

(limited to 'spec')

diff --git a/app/controllers/admin/accounts_controller.rb b/app/controllers/admin/accounts_controller.rb
index 2fa1dfe5f..68b6352f8 100644
--- a/app/controllers/admin/accounts_controller.rb
+++ b/app/controllers/admin/accounts_controller.rb
@@ -41,7 +41,7 @@ module Admin
 
     def reject
       authorize @account.user, :reject?
-      SuspendAccountService.new.call(@account, including_user: true, destroy: true, skip_distribution: true)
+      SuspendAccountService.new.call(@account, reserve_email: false, reserve_username: false)
       redirect_to admin_pending_accounts_path
     end
 
diff --git a/app/controllers/admin/report_notes_controller.rb b/app/controllers/admin/report_notes_controller.rb
index bcb3f2026..b816c5b5d 100644
--- a/app/controllers/admin/report_notes_controller.rb
+++ b/app/controllers/admin/report_notes_controller.rb
@@ -5,10 +5,10 @@ module Admin
     before_action :set_report_note, only: [:destroy]
 
     def create
-      authorize ReportNote, :create?
+      authorize :report_note, :create?
 
       @report_note = current_account.report_notes.new(resource_params)
-      @report = @report_note.report
+      @report      = @report_note.report
 
       if @report_note.save
         if params[:create_and_resolve]
@@ -26,9 +26,8 @@ module Admin
 
         redirect_to admin_report_path(@report), notice: I18n.t('admin.report_notes.created_msg')
       else
-        @report_notes = @report.notes.latest
-        @report_history = @report.history
-        @form = Form::StatusBatch.new
+        @report_notes = (@report.notes.latest + @report.history + @report.target_account.targeted_account_warnings.latest.custom).sort_by(&:created_at)
+        @form         = Form::StatusBatch.new
 
         render template: 'admin/reports/show'
       end
diff --git a/app/controllers/api/v1/admin/accounts_controller.rb b/app/controllers/api/v1/admin/accounts_controller.rb
index c306180ca..c35ea5ab2 100644
--- a/app/controllers/api/v1/admin/accounts_controller.rb
+++ b/app/controllers/api/v1/admin/accounts_controller.rb
@@ -58,7 +58,7 @@ class Api::V1::Admin::AccountsController < Api::BaseController
 
   def reject
     authorize @account.user, :reject?
-    SuspendAccountService.new.call(@account, including_user: true, destroy: true, skip_distribution: true)
+    SuspendAccountService.new.call(@account, reserve_email: false, reserve_username: false)
     render json: @account, serializer: REST::Admin::AccountSerializer
   end
 
diff --git a/app/lib/activitypub/activity/delete.rb b/app/lib/activitypub/activity/delete.rb
index 345060462..dc9ff580c 100644
--- a/app/lib/activitypub/activity/delete.rb
+++ b/app/lib/activitypub/activity/delete.rb
@@ -13,8 +13,7 @@ class ActivityPub::Activity::Delete < ActivityPub::Activity
 
   def delete_person
     lock_or_return("delete_in_progress:#{@account.id}") do
-      SuspendAccountService.new.call(@account)
-      @account.destroy!
+      SuspendAccountService.new.call(@account, reserve_username: false)
     end
   end
 
diff --git a/app/models/account.rb b/app/models/account.rb
index 8c9388b95..55fe53fae 100644
--- a/app/models/account.rb
+++ b/app/models/account.rb
@@ -115,6 +115,7 @@ class Account < ApplicationRecord
            :approved?,
            :pending?,
            :disabled?,
+           :unconfirmed_or_pending?,
            :role,
            :admin?,
            :moderator?,
diff --git a/app/models/admin/account_action.rb b/app/models/admin/account_action.rb
index c7da8b52c..b30a82369 100644
--- a/app/models/admin/account_action.rb
+++ b/app/models/admin/account_action.rb
@@ -83,19 +83,23 @@ class Admin::AccountAction
 
     # A log entry is only interesting if the warning contains
     # custom text from someone. Otherwise it's just noise.
+
     log_action(:create, warning) if warning.text.present?
   end
 
   def process_reports!
-    return if report_id.blank?
+    # If we're doing "mark as resolved" on a single report,
+    # then we want to keep other reports open in case they
+    # contain new actionable information.
+    #
+    # Otherwise, we will mark all unresolved reports about
+    # the account as resolved.
 
-    authorize(report, :update?)
+    reports.each { |report| authorize(report, :update?) }
 
-    if type == 'none'
+    reports.each do |report|
       log_action(:resolve, report)
       report.resolve!(current_account)
-    else
-      Report.where(target_account: target_account).unresolved.update_all(action_taken: true, action_taken_by_account_id: current_account.id)
     end
   end
 
@@ -141,6 +145,16 @@ class Admin::AccountAction
     @report.status_ids if @report && include_statuses
   end
 
+  def reports
+    @reports ||= begin
+      if type == 'none' && with_report?
+        [report]
+      else
+        Report.where(target_account: target_account).unresolved
+      end
+    end
+  end
+
   def warning_preset
     @warning_preset ||= AccountWarningPreset.find(warning_preset_id) if warning_preset_id.present?
   end
diff --git a/app/models/form/account_batch.rb b/app/models/form/account_batch.rb
index f1b7a4566..0b285fde9 100644
--- a/app/models/form/account_batch.rb
+++ b/app/models/form/account_batch.rb
@@ -69,6 +69,6 @@ class Form::AccountBatch
     records = accounts.includes(:user)
 
     records.each { |account| authorize(account.user, :reject?) }
-           .each { |account| SuspendAccountService.new.call(account, including_user: true, destroy: true, skip_distribution: true) }
+           .each { |account| SuspendAccountService.new.call(account, reserve_email: false, reserve_username: false) }
   end
 end
diff --git a/app/models/form/status_batch.rb b/app/models/form/status_batch.rb
index e09cc2594..c4943a7ea 100644
--- a/app/models/form/status_batch.rb
+++ b/app/models/form/status_batch.rb
@@ -35,7 +35,7 @@ class Form::StatusBatch
   def delete_statuses
     Status.where(id: status_ids).reorder(nil).find_each do |status|
       status.discard
-      RemovalWorker.perform_async(status.id, redraft: false)
+      RemovalWorker.perform_async(status.id, immediate: true)
       Tombstone.find_or_create_by(uri: status.uri, account: status.account, by_moderator: true)
       log_action :destroy, status
     end
diff --git a/app/models/report.rb b/app/models/report.rb
index 1e707ff1c..fb2e040ee 100644
--- a/app/models/report.rb
+++ b/app/models/report.rb
@@ -59,6 +59,7 @@ class Report < ApplicationRecord
   end
 
   def resolve!(acting_account)
+    RemovalWorker.push_bulk(Status.with_discarded.discarded.where(id: status_ids).pluck(:id)) { |status_id| [status_id, { immediate: true }] }
     update!(action_taken: true, action_taken_by_account_id: acting_account.id)
   end
 
diff --git a/app/models/status.rb b/app/models/status.rb
index 9cfaddcec..471bb03b4 100644
--- a/app/models/status.rb
+++ b/app/models/status.rb
@@ -214,6 +214,10 @@ class Status < ApplicationRecord
     !sensitive? && with_media?
   end
 
+  def reported?
+    @reported ||= Report.where(target_account: account).unresolved.where('? = ANY(status_ids)', id).exists?
+  end
+
   def emojis
     return @emojis if defined?(@emojis)
 
diff --git a/app/models/user.rb b/app/models/user.rb
index 95f1d8fc5..78b82a68f 100644
--- a/app/models/user.rb
+++ b/app/models/user.rb
@@ -171,6 +171,10 @@ class User < ApplicationRecord
     confirmed? && approved? && !disabled? && !account.suspended?
   end
 
+  def unconfirmed_or_pending?
+    !(confirmed? && approved?)
+  end
+
   def inactive_message
     !approved? ? :pending : super
   end
diff --git a/app/services/block_domain_service.rb b/app/services/block_domain_service.rb
index 0ec6be503..ae461abf2 100644
--- a/app/services/block_domain_service.rb
+++ b/app/services/block_domain_service.rb
@@ -53,7 +53,7 @@ class BlockDomainService < BaseService
 
   def suspend_accounts!
     blocked_domain_accounts.without_suspended.reorder(nil).find_each do |account|
-      SuspendAccountService.new.call(account, suspended_at: @domain_block.created_at)
+      SuspendAccountService.new.call(account, reserve_username: true, suspended_at: @domain_block.created_at)
     end
   end
 
diff --git a/app/services/remove_status_service.rb b/app/services/remove_status_service.rb
index 685c1d4bf..f9352ed3d 100644
--- a/app/services/remove_status_service.rb
+++ b/app/services/remove_status_service.rb
@@ -8,7 +8,8 @@ class RemoveStatusService < BaseService
   # @param   [Status] status
   # @param   [Hash] options
   # @option  [Boolean] :redraft
-  # @options [Boolean] :original_removed
+  # @option  [Boolean] :immediate
+  # @option [Boolean] :original_removed
   def call(status, **options)
     @payload  = Oj.dump(event: :delete, payload: status.id.to_s)
     @status   = status
@@ -31,7 +32,7 @@ class RemoveStatusService < BaseService
         remove_from_spam_check
         remove_media
 
-        @status.destroy!
+        @status.destroy! if @options[:immediate] || !@status.reported?
       else
         raise Mastodon::RaceConditionError
       end
@@ -150,7 +151,7 @@ class RemoveStatusService < BaseService
   end
 
   def remove_media
-    return if @options[:redraft]
+    return if @options[:redraft] || (!@options[:immediate] && @status.reported?)
 
     @status.media_attachments.destroy_all
   end
diff --git a/app/services/suspend_account_service.rb b/app/services/suspend_account_service.rb
index 85da7e921..ecc893931 100644
--- a/app/services/suspend_account_service.rb
+++ b/app/services/suspend_account_service.rb
@@ -15,7 +15,6 @@ class SuspendAccountService < BaseService
     favourites
     follow_requests
     list_accounts
-    media_attachments
     mute_relationships
     muted_by_relationships
     notifications
@@ -32,14 +31,26 @@ class SuspendAccountService < BaseService
     targeted_reports
   ).freeze
 
-  # Suspend an account and remove as much of its data as possible
+  # Suspend or remove an account and remove as much of its data
+  # as possible. If it's a local account and it has not been confirmed
+  # or never been approved, then side effects are skipped and both
+  # the user and account records are removed fully. Otherwise,
+  # it is controlled by options.
   # @param [Account]
   # @param [Hash] options
-  # @option [Boolean] :including_user Remove the user record as well
-  # @option [Boolean] :destroy Remove the account record instead of suspending
+  # @option [Boolean] :reserve_email Keep user record. Only applicable for local accounts
+  # @option [Boolean] :reserve_username Keep account record
+  # @option [Boolean] :skip_side_effects Side effects are ActivityPub and streaming API payloads
+  # @option [Time]    :suspended_at Only applicable when :reserve_username is true
   def call(account, **options)
     @account = account
-    @options = options
+    @options = { reserve_username: true, reserve_email: true }.merge(options)
+
+    if @account.local? && @account.user_unconfirmed_or_pending?
+      @options[:reserve_email]     = false
+      @options[:reserve_username]  = false
+      @options[:skip_side_effects] = true
+    end
 
     reject_follows!
     purge_user!
@@ -60,27 +71,39 @@ class SuspendAccountService < BaseService
   def purge_user!
     return if !@account.local? || @account.user.nil?
 
-    if @options[:including_user]
-      @options[:destroy] = true if !@account.user_confirmed? || @account.user_pending?
-      @account.user.destroy
-    else
+    if @options[:reserve_email]
       @account.user.disable!
       @account.user.invites.where(uses: 0).destroy_all
+    else
+      @account.user.destroy
     end
   end
 
   def purge_content!
-    distribute_delete_actor! if @account.local? && !@options[:skip_distribution]
+    distribute_delete_actor! if @account.local? && !@options[:skip_side_effects]
 
     @account.statuses.reorder(nil).find_in_batches do |statuses|
-      BatchedRemoveStatusService.new.call(statuses, skip_side_effects: @options[:destroy])
+      statuses.reject! { |status| reported_status_ids.include?(status.id) } if @options[:reserve_username]
+      BatchedRemoveStatusService.new.call(statuses, skip_side_effects: @options[:skip_side_effects])
+    end
+
+    @account.media_attachments.reorder(nil).find_each do |media_attachment|
+      next if @options[:reserve_username] && reported_status_ids.include?(media_attachment.status_id)
+
+      media_attachment.destroy
+    end
+
+    @account.polls.reorder(nil).find_each do |poll|
+      next if @options[:reserve_username] && reported_status_ids.include?(poll.status_id)
+
+      poll.destroy
     end
 
     associations_for_destruction.each do |association_name|
       destroy_all(@account.public_send(association_name))
     end
 
-    @account.destroy if @options[:destroy]
+    @account.destroy unless @options[:reserve_username]
   end
 
   def purge_profile!
@@ -88,11 +111,13 @@ class SuspendAccountService < BaseService
     # there is no point wasting time updating
     # its values first
 
-    return if @options[:destroy]
+    return unless @options[:reserve_username]
 
     @account.silenced_at      = nil
     @account.suspended_at     = @options[:suspended_at] || Time.now.utc
     @account.locked           = false
+    @account.memorial         = false
+    @account.discoverable     = false
     @account.display_name     = ''
     @account.note             = ''
     @account.fields           = []
@@ -100,6 +125,7 @@ class SuspendAccountService < BaseService
     @account.followers_count  = 0
     @account.following_count  = 0
     @account.moved_to_account = nil
+    @account.trust_level      = :untrusted
     @account.avatar.destroy
     @account.header.destroy
     @account.save!
@@ -135,11 +161,15 @@ class SuspendAccountService < BaseService
     Account.inboxes - delivery_inboxes
   end
 
+  def reported_status_ids
+    @reported_status_ids ||= Report.where(target_account: @account).unresolved.pluck(:status_ids).flatten.uniq
+  end
+
   def associations_for_destruction
-    if @options[:destroy]
-      ASSOCIATIONS_ON_SUSPEND + ASSOCIATIONS_ON_DESTROY
-    else
+    if @options[:reserve_username]
       ASSOCIATIONS_ON_SUSPEND
+    else
+      ASSOCIATIONS_ON_SUSPEND + ASSOCIATIONS_ON_DESTROY
     end
   end
 end
diff --git a/app/services/unallow_domain_service.rb b/app/services/unallow_domain_service.rb
index d4387c1a1..bd1ad328d 100644
--- a/app/services/unallow_domain_service.rb
+++ b/app/services/unallow_domain_service.rb
@@ -3,7 +3,7 @@
 class UnallowDomainService < BaseService
   def call(domain_allow)
     Account.where(domain: domain_allow.domain).find_each do |account|
-      SuspendAccountService.new.call(account, destroy: true)
+      SuspendAccountService.new.call(account, reserve_username: false)
     end
 
     domain_allow.destroy
diff --git a/app/workers/admin/suspension_worker.rb b/app/workers/admin/suspension_worker.rb
index ae8b24d8c..83c815efd 100644
--- a/app/workers/admin/suspension_worker.rb
+++ b/app/workers/admin/suspension_worker.rb
@@ -6,6 +6,6 @@ class Admin::SuspensionWorker
   sidekiq_options queue: 'pull'
 
   def perform(account_id, remove_user = false)
-    SuspendAccountService.new.call(Account.find(account_id), including_user: remove_user)
+    SuspendAccountService.new.call(Account.find(account_id), reserve_username: true, reserve_email: !remove_user)
   end
 end
diff --git a/lib/mastodon/accounts_cli.rb b/lib/mastodon/accounts_cli.rb
index b16bf2e38..a09a6ab04 100644
--- a/lib/mastodon/accounts_cli.rb
+++ b/lib/mastodon/accounts_cli.rb
@@ -185,7 +185,7 @@ module Mastodon
       end
 
       say("Deleting user with #{account.statuses_count} statuses, this might take a while...")
-      SuspendAccountService.new.call(account, including_user: true)
+      SuspendAccountService.new.call(account, reserve_email: false)
       say('OK', :green)
     end
 
@@ -239,7 +239,7 @@ module Mastodon
         end
 
         if [404, 410].include?(code)
-          SuspendAccountService.new.call(account, destroy: true) unless options[:dry_run]
+          SuspendAccountService.new.call(account, reserve_username: false) unless options[:dry_run]
           1
         else
           # Touch account even during dry run to avoid getting the account into the window again
diff --git a/lib/mastodon/domains_cli.rb b/lib/mastodon/domains_cli.rb
index c612c2d72..8e52de1c3 100644
--- a/lib/mastodon/domains_cli.rb
+++ b/lib/mastodon/domains_cli.rb
@@ -42,7 +42,7 @@ module Mastodon
       end
 
       processed, = parallelize_with_progress(scope) do |account|
-        SuspendAccountService.new.call(account, destroy: true) unless options[:dry_run]
+        SuspendAccountService.new.call(account, reserve_username: false, skip_side_effects: true) unless options[:dry_run]
       end
 
       DomainBlock.where(domain: domain).destroy_all unless options[:dry_run]
diff --git a/spec/controllers/admin/reported_statuses_controller_spec.rb b/spec/controllers/admin/reported_statuses_controller_spec.rb
index bd146b795..2a1598123 100644
--- a/spec/controllers/admin/reported_statuses_controller_spec.rb
+++ b/spec/controllers/admin/reported_statuses_controller_spec.rb
@@ -47,7 +47,7 @@ describe Admin::ReportedStatusesController do
       it 'removes a status' do
         allow(RemovalWorker).to receive(:perform_async)
         subject.call
-        expect(RemovalWorker).to have_received(:perform_async).with(status_ids.first, redraft: false)
+        expect(RemovalWorker).to have_received(:perform_async).with(status_ids.first, immediate: true)
       end
     end
 
diff --git a/spec/controllers/admin/statuses_controller_spec.rb b/spec/controllers/admin/statuses_controller_spec.rb
index 6b06343ef..d9690d83f 100644
--- a/spec/controllers/admin/statuses_controller_spec.rb
+++ b/spec/controllers/admin/statuses_controller_spec.rb
@@ -65,7 +65,7 @@ describe Admin::StatusesController do
       it 'removes a status' do
         allow(RemovalWorker).to receive(:perform_async)
         subject.call
-        expect(RemovalWorker).to have_received(:perform_async).with(status_ids.first, redraft: false)
+        expect(RemovalWorker).to have_received(:perform_async).with(status_ids.first, immediate: true)
       end
     end
 
diff --git a/spec/models/form/status_batch_spec.rb b/spec/models/form/status_batch_spec.rb
index f9c58c90f..68d84a737 100644
--- a/spec/models/form/status_batch_spec.rb
+++ b/spec/models/form/status_batch_spec.rb
@@ -41,12 +41,12 @@ describe Form::StatusBatch do
 
     it 'call RemovalWorker' do
       form.save
-      expect(RemovalWorker).to have_received(:perform_async).with(status.id, redraft: false)
+      expect(RemovalWorker).to have_received(:perform_async).with(status.id, immediate: true)
     end
 
     it 'do not call RemovalWorker' do
       form.save
-      expect(RemovalWorker).not_to have_received(:perform_async).with(another_status.id, redraft: false)
+      expect(RemovalWorker).not_to have_received(:perform_async).with(another_status.id, immediate: true)
     end
   end
 end
-- 
cgit 


From 0762258aec68f1d422a0ecdc29d637c5807f943a Mon Sep 17 00:00:00 2001
From: Eugen Rochko <eugen@zeonfederated.com>
Date: Fri, 13 Sep 2019 16:01:26 +0200
Subject: Fix hashtags being split by ZWNJ character (#11821)

Fix #11761
---
 app/models/tag.rb       | 5 +++--
 spec/models/tag_spec.rb | 4 ++++
 2 files changed, 7 insertions(+), 2 deletions(-)

(limited to 'spec')

diff --git a/app/models/tag.rb b/app/models/tag.rb
index 135e0a030..a6aed0d68 100644
--- a/app/models/tag.rb
+++ b/app/models/tag.rb
@@ -25,8 +25,9 @@ class Tag < ApplicationRecord
   has_many :featured_tags, dependent: :destroy, inverse_of: :tag
   has_one :account_tag_stat, dependent: :destroy
 
-  HASHTAG_NAME_RE = '([[:word:]_][[:word:]_·]*[[:alpha:]_·][[:word:]_·]*[[:word:]_])|([[:word:]_]*[[:alpha:]][[:word:]_]*)'
-  HASHTAG_RE = /(?:^|[^\/\)\w])#(#{HASHTAG_NAME_RE})/i
+  HASHTAG_SEPARATORS = "_\u00B7\u200c"
+  HASHTAG_NAME_RE    = "([[:word:]_][[:word:]#{HASHTAG_SEPARATORS}]*[[:alpha:]#{HASHTAG_SEPARATORS}][[:word:]#{HASHTAG_SEPARATORS}]*[[:word:]_])|([[:word:]_]*[[:alpha:]][[:word:]_]*)"
+  HASHTAG_RE         = /(?:^|[^\/\)\w])#(#{HASHTAG_NAME_RE})/i
 
   validates :name, presence: true, format: { with: /\A(#{HASHTAG_NAME_RE})\z/i }
   validate :validate_name_change, if: -> { !new_record? && name_changed? }
diff --git a/spec/models/tag_spec.rb b/spec/models/tag_spec.rb
index 2bb30fb57..df876593c 100644
--- a/spec/models/tag_spec.rb
+++ b/spec/models/tag_spec.rb
@@ -62,6 +62,10 @@ RSpec.describe Tag, type: :model do
       expect(subject.match('hello #one·two·three').to_s).to eq ' #one·two·three'
     end
 
+    it 'matches ZWNJ' do
+      expect(subject.match('just add #نرم‌افزار and').to_s).to eq ' #نرم‌افزار'
+    end
+
     it 'does not match middle dots at the start' do
       expect(subject.match('hello #·one·two·three')).to be_nil
     end
-- 
cgit 


From 18331fefa2246facc818226043b1f9cc67cf6c1a Mon Sep 17 00:00:00 2001
From: Eugen Rochko <eugen@zeonfederated.com>
Date: Fri, 13 Sep 2019 16:11:13 +0200
Subject: Remove deprecated `GET /api/v1/search` API (#11823)

Use `GET /api/v2/search` instead
---
 app/controllers/api/v1/search_controller.rb       | 32 -----------------------
 app/controllers/api/v2/search_controller.rb       | 28 ++++++++++++++++++--
 app/serializers/rest/search_serializer.rb         |  7 +----
 app/serializers/rest/v2/search_serializer.rb      |  7 -----
 config/routes.rb                                  |  2 --
 spec/controllers/api/v1/search_controller_spec.rb | 22 ----------------
 6 files changed, 27 insertions(+), 71 deletions(-)
 delete mode 100644 app/controllers/api/v1/search_controller.rb
 delete mode 100644 app/serializers/rest/v2/search_serializer.rb
 delete mode 100644 spec/controllers/api/v1/search_controller_spec.rb

(limited to 'spec')

diff --git a/app/controllers/api/v1/search_controller.rb b/app/controllers/api/v1/search_controller.rb
deleted file mode 100644
index 6131cbbb6..000000000
--- a/app/controllers/api/v1/search_controller.rb
+++ /dev/null
@@ -1,32 +0,0 @@
-# frozen_string_literal: true
-
-class Api::V1::SearchController < Api::BaseController
-  include Authorization
-
-  RESULTS_LIMIT = 20
-
-  before_action -> { doorkeeper_authorize! :read, :'read:search' }
-  before_action :require_user!
-
-  respond_to :json
-
-  def index
-    @search = Search.new(search_results)
-    render json: @search, serializer: REST::SearchSerializer
-  end
-
-  private
-
-  def search_results
-    SearchService.new.call(
-      params[:q],
-      current_account,
-      limit_param(RESULTS_LIMIT),
-      search_params.merge(resolve: truthy_param?(:resolve))
-    )
-  end
-
-  def search_params
-    params.permit(:type, :offset, :min_id, :max_id, :account_id)
-  end
-end
diff --git a/app/controllers/api/v2/search_controller.rb b/app/controllers/api/v2/search_controller.rb
index 9aa6edc69..c14cd22d7 100644
--- a/app/controllers/api/v2/search_controller.rb
+++ b/app/controllers/api/v2/search_controller.rb
@@ -1,8 +1,32 @@
 # frozen_string_literal: true
 
-class Api::V2::SearchController < Api::V1::SearchController
+class Api::V2::SearchController < Api::BaseController
+  include Authorization
+
+  RESULTS_LIMIT = 20
+
+  before_action -> { doorkeeper_authorize! :read, :'read:search' }
+  before_action :require_user!
+
+  respond_to :json
+
   def index
     @search = Search.new(search_results)
-    render json: @search, serializer: REST::V2::SearchSerializer
+    render json: @search, serializer: REST::SearchSerializer
+  end
+
+  private
+
+  def search_results
+    SearchService.new.call(
+      params[:q],
+      current_account,
+      limit_param(RESULTS_LIMIT),
+      search_params.merge(resolve: truthy_param?(:resolve))
+    )
+  end
+
+  def search_params
+    params.permit(:type, :offset, :min_id, :max_id, :account_id)
   end
 end
diff --git a/app/serializers/rest/search_serializer.rb b/app/serializers/rest/search_serializer.rb
index 157f543ae..ee9b421eb 100644
--- a/app/serializers/rest/search_serializer.rb
+++ b/app/serializers/rest/search_serializer.rb
@@ -1,12 +1,7 @@
 # frozen_string_literal: true
 
 class REST::SearchSerializer < ActiveModel::Serializer
-  attributes :hashtags
-
   has_many :accounts, serializer: REST::AccountSerializer
   has_many :statuses, serializer: REST::StatusSerializer
-
-  def hashtags
-    object.hashtags.map(&:name)
-  end
+  has_many :hashtags, serializer: REST::TagSerializer
 end
diff --git a/app/serializers/rest/v2/search_serializer.rb b/app/serializers/rest/v2/search_serializer.rb
deleted file mode 100644
index cdb6b3a53..000000000
--- a/app/serializers/rest/v2/search_serializer.rb
+++ /dev/null
@@ -1,7 +0,0 @@
-# frozen_string_literal: true
-
-class REST::V2::SearchSerializer < ActiveModel::Serializer
-  has_many :accounts, serializer: REST::AccountSerializer
-  has_many :statuses, serializer: REST::StatusSerializer
-  has_many :hashtags, serializer: REST::TagSerializer
-end
diff --git a/config/routes.rb b/config/routes.rb
index d22a9e56a..a4dee2842 100644
--- a/config/routes.rb
+++ b/config/routes.rb
@@ -311,8 +311,6 @@ Rails.application.routes.draw do
         end
       end
 
-      get '/search', to: 'search#index', as: :search
-
       resources :media,        only: [:create, :update]
       resources :blocks,       only: [:index]
       resources :mutes,        only: [:index]
diff --git a/spec/controllers/api/v1/search_controller_spec.rb b/spec/controllers/api/v1/search_controller_spec.rb
deleted file mode 100644
index c9e544cc7..000000000
--- a/spec/controllers/api/v1/search_controller_spec.rb
+++ /dev/null
@@ -1,22 +0,0 @@
-# frozen_string_literal: true
-
-require 'rails_helper'
-
-RSpec.describe Api::V1::SearchController, type: :controller do
-  render_views
-
-  let(:user)  { Fabricate(:user, account: Fabricate(:account, username: 'alice')) }
-  let(:token) { Fabricate(:accessible_access_token, resource_owner_id: user.id, scopes: 'read:search') }
-
-  before do
-    allow(controller).to receive(:doorkeeper_token) { token }
-  end
-
-  describe 'GET #index' do
-    it 'returns http success' do
-      get :index, params: { q: 'test' }
-
-      expect(response).to have_http_status(200)
-    end
-  end
-end
-- 
cgit 


From b83e2df6b59ccd7cbe8f9145e06b75547dc1101a Mon Sep 17 00:00:00 2001
From: Thibaut Girka <thib@sitedethib.com>
Date: Fri, 13 Sep 2019 20:18:25 +0200
Subject: Fix tests being broken because of different default setting

---
 spec/controllers/api/v1/timelines/public_controller_spec.rb | 4 ++++
 1 file changed, 4 insertions(+)

(limited to 'spec')

diff --git a/spec/controllers/api/v1/timelines/public_controller_spec.rb b/spec/controllers/api/v1/timelines/public_controller_spec.rb
index 737aedba6..b8e9d8674 100644
--- a/spec/controllers/api/v1/timelines/public_controller_spec.rb
+++ b/spec/controllers/api/v1/timelines/public_controller_spec.rb
@@ -44,6 +44,10 @@ describe Api::V1::Timelines::PublicController do
   context 'without a user context' do
     let(:token) { Fabricate(:accessible_access_token, resource_owner_id: nil) }
 
+    before do
+      Setting.timeline_preview = true
+    end
+
     describe 'GET #show' do
       it 'returns http success' do
         get :show
-- 
cgit 


From c707ef49d9b13932f4d98c127ec3148a5cdc3479 Mon Sep 17 00:00:00 2001
From: Eugen Rochko <eugen@zeonfederated.com>
Date: Sun, 15 Sep 2019 21:08:39 +0200
Subject: Fix 2FA challenge and password challenge for non-database users
 (#11831)

* Fix 2FA challenge not appearing for non-database users

Fix #11685

* Fix account deletion not working when using external login

Fix #11691
---
 app/controllers/auth/sessions_controller.rb       | 61 ++++++++++-------------
 app/controllers/settings/deletes_controller.rb    | 25 +++++++---
 app/models/form/delete_confirmation.rb            |  2 +-
 app/views/settings/deletes/show.html.haml         |  5 +-
 config/initializers/devise.rb                     |  7 ++-
 config/locales/en.yml                             |  3 +-
 spec/controllers/auth/sessions_controller_spec.rb | 24 +++------
 7 files changed, 66 insertions(+), 61 deletions(-)

(limited to 'spec')

diff --git a/app/controllers/auth/sessions_controller.rb b/app/controllers/auth/sessions_controller.rb
index 7e6dbf19e..3e93b2e68 100644
--- a/app/controllers/auth/sessions_controller.rb
+++ b/app/controllers/auth/sessions_controller.rb
@@ -8,8 +8,6 @@ class Auth::SessionsController < Devise::SessionsController
   skip_before_action :require_no_authentication, only: [:create]
   skip_before_action :require_functional!
 
-  prepend_before_action :authenticate_with_two_factor, if: :two_factor_enabled?, only: [:create]
-
   before_action :set_instance_presenter, only: [:new]
   before_action :set_body_classes
 
@@ -22,9 +20,22 @@ class Auth::SessionsController < Devise::SessionsController
   end
 
   def create
-    super do |resource|
-      remember_me(resource)
-      flash.delete(:notice)
+    self.resource = begin
+      if user_params[:email].blank? && session[:otp_user_id].present?
+        User.find(session[:otp_user_id])
+      else
+        warden.authenticate!(auth_options)
+      end
+    end
+
+    if resource.otp_required_for_login?
+      if user_params[:otp_attempt].present? && session[:otp_user_id].present?
+        authenticate_with_two_factor_via_otp(resource)
+      else
+        prompt_for_two_factor(resource)
+      end
+    else
+      authenticate_and_respond(resource)
     end
   end
 
@@ -37,18 +48,6 @@ class Auth::SessionsController < Devise::SessionsController
 
   protected
 
-  def find_user
-    if session[:otp_user_id]
-      User.find(session[:otp_user_id])
-    elsif user_params[:email]
-      if use_seamless_external_login? && Devise.check_at_sign && user_params[:email].index('@').nil?
-        User.joins(:account).find_by(accounts: { username: user_params[:email] })
-      else
-        User.find_for_authentication(email: user_params[:email])
-      end
-    end
-  end
-
   def user_params
     params.require(:user).permit(:email, :password, :otp_attempt)
   end
@@ -71,32 +70,17 @@ class Auth::SessionsController < Devise::SessionsController
     super
   end
 
-  def two_factor_enabled?
-    find_user.try(:otp_required_for_login?)
-  end
-
   def valid_otp_attempt?(user)
     user.validate_and_consume_otp!(user_params[:otp_attempt]) ||
       user.invalidate_otp_backup_code!(user_params[:otp_attempt])
-  rescue OpenSSL::Cipher::CipherError => _error
+  rescue OpenSSL::Cipher::CipherError
     false
   end
 
-  def authenticate_with_two_factor
-    user = self.resource = find_user
-
-    if user_params[:otp_attempt].present? && session[:otp_user_id]
-      authenticate_with_two_factor_via_otp(user)
-    elsif user&.valid_password?(user_params[:password])
-      prompt_for_two_factor(user)
-    end
-  end
-
   def authenticate_with_two_factor_via_otp(user)
     if valid_otp_attempt?(user)
       session.delete(:otp_user_id)
-      remember_me(user)
-      sign_in(user)
+      authenticate_and_respond(user)
     else
       flash.now[:alert] = I18n.t('users.invalid_otp_token')
       prompt_for_two_factor(user)
@@ -108,6 +92,13 @@ class Auth::SessionsController < Devise::SessionsController
     render :two_factor
   end
 
+  def authenticate_and_respond(user)
+    sign_in(user)
+    remember_me(user)
+
+    respond_with user, location: after_sign_in_path_for(user)
+  end
+
   private
 
   def set_instance_presenter
@@ -120,9 +111,11 @@ class Auth::SessionsController < Devise::SessionsController
 
   def home_paths(resource)
     paths = [about_path]
+
     if single_user_mode? && resource.is_a?(User)
       paths << short_account_path(username: resource.account)
     end
+
     paths
   end
 
diff --git a/app/controllers/settings/deletes_controller.rb b/app/controllers/settings/deletes_controller.rb
index 97fe4d328..15a59c999 100644
--- a/app/controllers/settings/deletes_controller.rb
+++ b/app/controllers/settings/deletes_controller.rb
@@ -14,12 +14,11 @@ class Settings::DeletesController < Settings::BaseController
   end
 
   def destroy
-    if current_user.valid_password?(delete_params[:password])
-      Admin::SuspensionWorker.perform_async(current_user.account_id, true)
-      sign_out
+    if challenge_passed?
+      destroy_account!
       redirect_to new_user_session_path, notice: I18n.t('deletes.success_msg')
     else
-      redirect_to settings_delete_path, alert: I18n.t('deletes.bad_password_msg')
+      redirect_to settings_delete_path, alert: I18n.t('deletes.challenge_not_passed')
     end
   end
 
@@ -29,11 +28,25 @@ class Settings::DeletesController < Settings::BaseController
     redirect_to root_path unless Setting.open_deletion
   end
 
-  def delete_params
-    params.require(:form_delete_confirmation).permit(:password)
+  def resource_params
+    params.require(:form_delete_confirmation).permit(:password, :username)
   end
 
   def require_not_suspended!
     forbidden if current_account.suspended?
   end
+
+  def challenge_passed?
+    if current_user.encrypted_password.blank?
+      current_account.username == resource_params[:username]
+    else
+      current_user.valid_password?(resource_params[:password])
+    end
+  end
+
+  def destroy_account!
+    current_account.suspend!
+    Admin::SuspensionWorker.perform_async(current_user.account_id, true)
+    sign_out
+  end
 end
diff --git a/app/models/form/delete_confirmation.rb b/app/models/form/delete_confirmation.rb
index 0884a09b8..99d04b331 100644
--- a/app/models/form/delete_confirmation.rb
+++ b/app/models/form/delete_confirmation.rb
@@ -3,5 +3,5 @@
 class Form::DeleteConfirmation
   include ActiveModel::Model
 
-  attr_accessor :password
+  attr_accessor :password, :username
 end
diff --git a/app/views/settings/deletes/show.html.haml b/app/views/settings/deletes/show.html.haml
index 6e2ff31c5..08792e0af 100644
--- a/app/views/settings/deletes/show.html.haml
+++ b/app/views/settings/deletes/show.html.haml
@@ -20,7 +20,10 @@
 
   %hr.spacer/
 
-  = f.input :password, wrapper: :with_block_label, input_html: { :autocomplete => 'off' }, hint: t('deletes.confirm_password')
+  - if current_user.encrypted_password.present?
+    = f.input :password, wrapper: :with_block_label, input_html: { :autocomplete => 'off' }, hint: t('deletes.confirm_password')
+  - else
+    = f.input :username, wrapper: :with_block_label, input_html: { :autocomplete => 'off' }, hint: t('deletes.confirm_username')
 
   .actions
     = f.button :button, t('deletes.proceed'), type: :submit, class: 'negative'
diff --git a/config/initializers/devise.rb b/config/initializers/devise.rb
index cd9bacf68..311583820 100644
--- a/config/initializers/devise.rb
+++ b/config/initializers/devise.rb
@@ -71,10 +71,13 @@ end
 
 Devise.setup do |config|
   config.warden do |manager|
+    manager.default_strategies(scope: :user).unshift :database_authenticatable
     manager.default_strategies(scope: :user).unshift :ldap_authenticatable if Devise.ldap_authentication
     manager.default_strategies(scope: :user).unshift :pam_authenticatable  if Devise.pam_authentication
-    manager.default_strategies(scope: :user).unshift :two_factor_authenticatable
-    manager.default_strategies(scope: :user).unshift :two_factor_backupable
+
+    # We handle 2FA in our own sessions controller so this gets in the way
+    manager.default_strategies(scope: :user).delete :two_factor_backupable
+    manager.default_strategies(scope: :user).delete :two_factor_authenticatable
   end
 
   # The secret key used by Devise. Devise uses this key to generate
diff --git a/config/locales/en.yml b/config/locales/en.yml
index 0a5ca31c1..8c9fe89f8 100644
--- a/config/locales/en.yml
+++ b/config/locales/en.yml
@@ -632,8 +632,9 @@ en:
       x_months: "%{count}mo"
       x_seconds: "%{count}s"
   deletes:
-    bad_password_msg: The password you entered was incorrect
+    challenge_not_passed: The information you entered was not correct
     confirm_password: Enter your current password to verify your identity
+    confirm_username: Enter your username to confirm the procedure
     proceed: Delete account
     success_msg: Your account was successfully deleted
     warning:
diff --git a/spec/controllers/auth/sessions_controller_spec.rb b/spec/controllers/auth/sessions_controller_spec.rb
index 87ef4f2bb..7ed5edde0 100644
--- a/spec/controllers/auth/sessions_controller_spec.rb
+++ b/spec/controllers/auth/sessions_controller_spec.rb
@@ -5,11 +5,11 @@ require 'rails_helper'
 RSpec.describe Auth::SessionsController, type: :controller do
   render_views
 
-  describe 'GET #new' do
-    before do
-      request.env['devise.mapping'] = Devise.mappings[:user]
-    end
+  before do
+    request.env['devise.mapping'] = Devise.mappings[:user]
+  end
 
+  describe 'GET #new' do
     it 'returns http success' do
       get :new
       expect(response).to have_http_status(200)
@@ -19,10 +19,6 @@ RSpec.describe Auth::SessionsController, type: :controller do
   describe 'DELETE #destroy' do
     let(:user) { Fabricate(:user) }
 
-    before do
-      request.env['devise.mapping'] = Devise.mappings[:user]
-    end
-
     context 'with a regular user' do
       it 'redirects to home after sign out' do
         sign_in(user, scope: :user)
@@ -51,10 +47,6 @@ RSpec.describe Auth::SessionsController, type: :controller do
   end
 
   describe 'POST #create' do
-    before do
-      request.env['devise.mapping'] = Devise.mappings[:user]
-    end
-
     context 'using PAM authentication', if: ENV['PAM_ENABLED'] == 'true' do
       context 'using a valid password' do
         before do
@@ -191,11 +183,11 @@ RSpec.describe Auth::SessionsController, type: :controller do
     end
 
     context 'using two-factor authentication' do
-      let(:user) do
-        Fabricate(:user, email: 'x@y.com', password: 'abcdefgh',
-                         otp_required_for_login: true, otp_secret: User.generate_otp_secret(32))
+      let!(:user) do
+        Fabricate(:user, email: 'x@y.com', password: 'abcdefgh', otp_required_for_login: true, otp_secret: User.generate_otp_secret(32))
       end
-      let(:recovery_codes) do
+
+      let!(:recovery_codes) do
         codes = user.generate_otp_backup_codes!
         user.save
         return codes
-- 
cgit 


From a4b60e9ba4874b9ab427bec41d8b2cd252ec4782 Mon Sep 17 00:00:00 2001
From: Eugen Rochko <eugen@zeonfederated.com>
Date: Wed, 18 Sep 2019 02:48:40 +0200
Subject: Fix TOTP codes not being filtered from logs during enabling/disabling
 (#11877)

Not a serious issue because they are meaningless past single use
---
 .../settings/two_factor_authentication/confirmations_controller.rb  | 4 ++--
 app/controllers/settings/two_factor_authentications_controller.rb   | 6 +++---
 app/models/form/two_factor_confirmation.rb                          | 2 +-
 .../settings/two_factor_authentication/confirmations/new.html.haml  | 2 +-
 app/views/settings/two_factor_authentications/show.html.haml        | 2 +-
 .../two_factor_authentication/confirmations_controller_spec.rb      | 6 +++---
 .../settings/two_factor_authentications_controller_spec.rb          | 4 ++--
 7 files changed, 13 insertions(+), 13 deletions(-)

(limited to 'spec')

diff --git a/app/controllers/settings/two_factor_authentication/confirmations_controller.rb b/app/controllers/settings/two_factor_authentication/confirmations_controller.rb
index 3145e092d..46c90bf74 100644
--- a/app/controllers/settings/two_factor_authentication/confirmations_controller.rb
+++ b/app/controllers/settings/two_factor_authentication/confirmations_controller.rb
@@ -15,7 +15,7 @@ module Settings
       end
 
       def create
-        if current_user.validate_and_consume_otp!(confirmation_params[:code])
+        if current_user.validate_and_consume_otp!(confirmation_params[:otp_attempt])
           flash.now[:notice] = I18n.t('two_factor_authentication.enabled_success')
 
           current_user.otp_required_for_login = true
@@ -33,7 +33,7 @@ module Settings
       private
 
       def confirmation_params
-        params.require(:form_two_factor_confirmation).permit(:code)
+        params.require(:form_two_factor_confirmation).permit(:otp_attempt)
       end
 
       def prepare_two_factor_form
diff --git a/app/controllers/settings/two_factor_authentications_controller.rb b/app/controllers/settings/two_factor_authentications_controller.rb
index 6904076e4..c93b17577 100644
--- a/app/controllers/settings/two_factor_authentications_controller.rb
+++ b/app/controllers/settings/two_factor_authentications_controller.rb
@@ -34,7 +34,7 @@ module Settings
     private
 
     def confirmation_params
-      params.require(:form_two_factor_confirmation).permit(:code)
+      params.require(:form_two_factor_confirmation).permit(:otp_attempt)
     end
 
     def verify_otp_required
@@ -42,8 +42,8 @@ module Settings
     end
 
     def acceptable_code?
-      current_user.validate_and_consume_otp!(confirmation_params[:code]) ||
-        current_user.invalidate_otp_backup_code!(confirmation_params[:code])
+      current_user.validate_and_consume_otp!(confirmation_params[:otp_attempt]) ||
+        current_user.invalidate_otp_backup_code!(confirmation_params[:otp_attempt])
     end
   end
 end
diff --git a/app/models/form/two_factor_confirmation.rb b/app/models/form/two_factor_confirmation.rb
index b8cf76d05..27ada6533 100644
--- a/app/models/form/two_factor_confirmation.rb
+++ b/app/models/form/two_factor_confirmation.rb
@@ -3,5 +3,5 @@
 class Form::TwoFactorConfirmation
   include ActiveModel::Model
 
-  attr_accessor :code
+  attr_accessor :otp_attempt
 end
diff --git a/app/views/settings/two_factor_authentication/confirmations/new.html.haml b/app/views/settings/two_factor_authentication/confirmations/new.html.haml
index e64155299..86cf1f695 100644
--- a/app/views/settings/two_factor_authentication/confirmations/new.html.haml
+++ b/app/views/settings/two_factor_authentication/confirmations/new.html.haml
@@ -12,7 +12,7 @@
       %samp.qr-alternative__code= current_user.otp_secret.scan(/.{4}/).join(' ')
 
   .fields-group
-    = f.input :code, wrapper: :with_label, hint: t('two_factor_authentication.code_hint'), label: t('simple_form.labels.defaults.otp_attempt'), input_html: { :autocomplete => 'off' }, required: true
+    = f.input :otp_attempt, wrapper: :with_label, hint: t('two_factor_authentication.code_hint'), label: t('simple_form.labels.defaults.otp_attempt'), input_html: { :autocomplete => 'off' }, required: true
 
   .actions
     = f.button :button, t('two_factor_authentication.enable'), type: :submit
diff --git a/app/views/settings/two_factor_authentications/show.html.haml b/app/views/settings/two_factor_authentications/show.html.haml
index 259bcd1ef..93509e022 100644
--- a/app/views/settings/two_factor_authentications/show.html.haml
+++ b/app/views/settings/two_factor_authentications/show.html.haml
@@ -10,7 +10,7 @@
   %hr/
 
   = simple_form_for @confirmation, url: settings_two_factor_authentication_path, method: :delete do |f|
-    = f.input :code, wrapper: :with_label, hint: t('two_factor_authentication.code_hint'), label: t('simple_form.labels.defaults.otp_attempt'), input_html: { :autocomplete => 'off' }, required: true
+    = f.input :otp_attempt, wrapper: :with_label, hint: t('two_factor_authentication.code_hint'), label: t('simple_form.labels.defaults.otp_attempt'), input_html: { :autocomplete => 'off' }, required: true
 
     .actions
       = f.button :button, t('two_factor_authentication.disable'), type: :submit
diff --git a/spec/controllers/settings/two_factor_authentication/confirmations_controller_spec.rb b/spec/controllers/settings/two_factor_authentication/confirmations_controller_spec.rb
index 2222a7559..2e5a9325c 100644
--- a/spec/controllers/settings/two_factor_authentication/confirmations_controller_spec.rb
+++ b/spec/controllers/settings/two_factor_authentication/confirmations_controller_spec.rb
@@ -68,7 +68,7 @@ describe Settings::TwoFactorAuthentication::ConfirmationsController do
             true
           end
 
-          post :create, params: { form_two_factor_confirmation: { code: '123456' } }
+          post :create, params: { form_two_factor_confirmation: { otp_attempt: '123456' } }
 
           expect(assigns(:recovery_codes)).to eq otp_backup_codes
           expect(flash[:notice]).to eq 'Two-factor authentication successfully enabled'
@@ -85,7 +85,7 @@ describe Settings::TwoFactorAuthentication::ConfirmationsController do
             false
           end
 
-          post :create, params: { form_two_factor_confirmation: { code: '123456' } }
+          post :create, params: { form_two_factor_confirmation: { otp_attempt: '123456' } }
         end
 
         it 'renders the new view' do
@@ -99,7 +99,7 @@ describe Settings::TwoFactorAuthentication::ConfirmationsController do
 
     context 'when not signed in' do
       it 'redirects if not signed in' do
-        post :create, params: { form_two_factor_confirmation: { code: '123456' } }
+        post :create, params: { form_two_factor_confirmation: { otp_attempt: '123456' } }
         expect(response).to redirect_to('/auth/sign_in')
       end
     end
diff --git a/spec/controllers/settings/two_factor_authentications_controller_spec.rb b/spec/controllers/settings/two_factor_authentications_controller_spec.rb
index f7c628756..922231ded 100644
--- a/spec/controllers/settings/two_factor_authentications_controller_spec.rb
+++ b/spec/controllers/settings/two_factor_authentications_controller_spec.rb
@@ -91,7 +91,7 @@ describe Settings::TwoFactorAuthenticationsController do
           true
         end
 
-        post :destroy, params: { form_two_factor_confirmation: { code: '123456' } }
+        post :destroy, params: { form_two_factor_confirmation: { otp_attempt: '123456' } }
 
         expect(response).to redirect_to(settings_two_factor_authentication_path)
         user.reload
@@ -105,7 +105,7 @@ describe Settings::TwoFactorAuthenticationsController do
           false
         end
 
-        post :destroy, params: { form_two_factor_confirmation: { code: '057772' } }
+        post :destroy, params: { form_two_factor_confirmation: { otp_attempt: '057772' } }
 
         user.reload
         expect(user.otp_required_for_login).to eq(true)
-- 
cgit 


From 4f6af87906175d9ea802ef0c6f050388eac890fa Mon Sep 17 00:00:00 2001
From: Eugen Rochko <eugen@zeonfederated.com>
Date: Wed, 18 Sep 2019 12:53:13 +0200
Subject: Change spam check to apply to local accounts and add a threshold
 (#11806)

Instead of detecting spam on first duplicate message, add a
threshold of 5 such messages to reduce false positives
---
 app/lib/activitypub/activity/create.rb   | 10 +------
 app/lib/spam_check.rb                    | 46 ++++++++++++++++++++++++++------
 app/services/process_mentions_service.rb |  5 ++++
 spec/lib/spam_check_spec.rb              | 34 ++++++++++++++---------
 4 files changed, 66 insertions(+), 29 deletions(-)

(limited to 'spec')

diff --git a/app/lib/activitypub/activity/create.rb b/app/lib/activitypub/activity/create.rb
index dea7fd43c..e69193b71 100644
--- a/app/lib/activitypub/activity/create.rb
+++ b/app/lib/activitypub/activity/create.rb
@@ -408,15 +408,7 @@ class ActivityPub::Activity::Create < ActivityPub::Activity
   end
 
   def check_for_spam
-    spam_check = SpamCheck.new(@status)
-
-    return if spam_check.skip?
-
-    if spam_check.spam?
-      spam_check.flag!
-    else
-      spam_check.remember!
-    end
+    SpamCheck.perform(@status)
   end
 
   def forward_for_reply
diff --git a/app/lib/spam_check.rb b/app/lib/spam_check.rb
index 0cf1b8790..441697364 100644
--- a/app/lib/spam_check.rb
+++ b/app/lib/spam_check.rb
@@ -4,9 +4,25 @@ class SpamCheck
   include Redisable
   include ActionView::Helpers::TextHelper
 
+  # Threshold over which two Nilsimsa values are considered
+  # to refer to the same text
   NILSIMSA_COMPARE_THRESHOLD = 95
-  NILSIMSA_MIN_SIZE          = 10
-  EXPIRE_SET_AFTER           = 1.week.seconds
+
+  # Nilsimsa doesn't work well on small inputs, so below
+  # this size, we check only for exact matches with MD5
+  NILSIMSA_MIN_SIZE = 10
+
+  # How long to keep the trail of digests between updates,
+  # there is no reason to store it forever
+  EXPIRE_SET_AFTER = 1.week.seconds
+
+  # How many digests to keep in an account's trail. If it's
+  # too small, spam could rotate around different message templates
+  MAX_TRAIL_SIZE = 10
+
+  # How many detected duplicates to allow through before
+  # considering the message as spam
+  THRESHOLD = 5
 
   def initialize(status)
     @account = status.account
@@ -21,9 +37,9 @@ class SpamCheck
     if insufficient_data?
       false
     elsif nilsimsa?
-      any_other_digest?('nilsimsa') { |_, other_digest| nilsimsa_compare_value(digest, other_digest) >= NILSIMSA_COMPARE_THRESHOLD }
+      digests_over_threshold?('nilsimsa') { |_, other_digest| nilsimsa_compare_value(digest, other_digest) >= NILSIMSA_COMPARE_THRESHOLD }
     else
-      any_other_digest?('md5') { |_, other_digest| other_digest == digest }
+      digests_over_threshold?('md5') { |_, other_digest| other_digest == digest }
     end
   end
 
@@ -38,7 +54,7 @@ class SpamCheck
     # get the correct status ID back, we have to save it in the string value
 
     redis.zadd(redis_key, @status.id, digest_with_algorithm)
-    redis.zremrangebyrank(redis_key, '0', '-10')
+    redis.zremrangebyrank(redis_key, 0, -(MAX_TRAIL_SIZE + 1))
     redis.expire(redis_key, EXPIRE_SET_AFTER)
   end
 
@@ -78,6 +94,20 @@ class SpamCheck
     end
   end
 
+  class << self
+    def perform(status)
+      spam_check = new(status)
+
+      return if spam_check.skip?
+
+      if spam_check.spam?
+        spam_check.flag!
+      else
+        spam_check.remember!
+      end
+    end
+  end
+
   private
 
   def disabled?
@@ -149,14 +179,14 @@ class SpamCheck
     redis.zrange(redis_key, 0, -1)
   end
 
-  def any_other_digest?(filter_algorithm)
-    other_digests.any? do |record|
+  def digests_over_threshold?(filter_algorithm)
+    other_digests.select do |record|
       algorithm, other_digest, status_id = record.split(':')
 
       next unless algorithm == filter_algorithm
 
       yield algorithm, other_digest, status_id
-    end
+    end.size >= THRESHOLD
   end
 
   def matching_status_ids
diff --git a/app/services/process_mentions_service.rb b/app/services/process_mentions_service.rb
index 90dca9740..2f7a9e985 100644
--- a/app/services/process_mentions_service.rb
+++ b/app/services/process_mentions_service.rb
@@ -33,6 +33,7 @@ class ProcessMentionsService < BaseService
     end
 
     status.save!
+    check_for_spam(status)
 
     mentions.each { |mention| create_notification(mention) }
   end
@@ -61,4 +62,8 @@ class ProcessMentionsService < BaseService
   def resolve_account_service
     ResolveAccountService.new
   end
+
+  def check_for_spam(status)
+    SpamCheck.perform(status)
+  end
 end
diff --git a/spec/lib/spam_check_spec.rb b/spec/lib/spam_check_spec.rb
index 9e0989216..4cae46111 100644
--- a/spec/lib/spam_check_spec.rb
+++ b/spec/lib/spam_check_spec.rb
@@ -86,23 +86,33 @@ RSpec.describe SpamCheck do
     end
 
     it 'returns true for duplicate statuses to the same recipient' do
-      status1 = status_with_html('@alice Hello')
-      described_class.new(status1).remember!
+      described_class::THRESHOLD.times do
+        status1 = status_with_html('@alice Hello')
+        described_class.new(status1).remember!
+      end
+
       status2 = status_with_html('@alice Hello')
       expect(described_class.new(status2).spam?).to be true
     end
 
     it 'returns true for duplicate statuses to different recipients' do
-      status1 = status_with_html('@alice Hello')
-      described_class.new(status1).remember!
+      described_class::THRESHOLD.times do
+        status1 = status_with_html('@alice Hello')
+        described_class.new(status1).remember!
+      end
+
       status2 = status_with_html('@bob Hello')
       expect(described_class.new(status2).spam?).to be true
     end
 
     it 'returns true for nearly identical statuses with random numbers' do
       source_text = 'Sodium, atomic number 11, was first isolated by Humphry Davy in 1807. A chemical component of salt, he named it Na in honor of the saltiest region on earth, North America.'
-      status1 = status_with_html('@alice ' + source_text + ' 1234')
-      described_class.new(status1).remember!
+
+      described_class::THRESHOLD.times do
+        status1 = status_with_html('@alice ' + source_text + ' 1234')
+        described_class.new(status1).remember!
+      end
+
       status2 = status_with_html('@bob ' + source_text + ' 9568')
       expect(described_class.new(status2).spam?).to be true
     end
@@ -140,9 +150,9 @@ RSpec.describe SpamCheck do
     let(:redis_key) { spam_check.send(:redis_key) }
 
     it 'remembers' do
-      expect do
-        spam_check.remember!
-      end.to change { Redis.current.exists(redis_key) }.from(false).to(true)
+      expect(Redis.current.exists(redis_key)).to be true
+      spam_check.remember!
+      expect(Redis.current.exists(redis_key)).to be true
     end
   end
 
@@ -156,9 +166,9 @@ RSpec.describe SpamCheck do
     end
 
     it 'resets' do
-      expect do
-        spam_check.reset!
-      end.to change { Redis.current.exists(redis_key) }.from(true).to(false)
+      expect(Redis.current.exists(redis_key)).to be true
+      spam_check.reset!
+      expect(Redis.current.exists(redis_key)).to be false
     end
   end
 
-- 
cgit 


From e1066cd4319a220d5be16e51ffaf5236a2f6e866 Mon Sep 17 00:00:00 2001
From: Eugen Rochko <eugen@zeonfederated.com>
Date: Wed, 18 Sep 2019 16:37:27 +0200
Subject: Add password challenge to 2FA settings, e-mail notifications (#11878)

Fix #3961
---
 .../admin/two_factor_authentications_controller.rb |   1 +
 app/controllers/auth/challenges_controller.rb      |  22 ++++
 app/controllers/auth/sessions_controller.rb        |   1 +
 app/controllers/concerns/challengable_concern.rb   |  65 ++++++++++++
 .../confirmations_controller.rb                    |   5 +
 .../recovery_codes_controller.rb                   |   6 ++
 .../two_factor_authentications_controller.rb       |   4 +
 app/javascript/styles/mastodon/admin.scss          |  43 ++++----
 app/javascript/styles/mastodon/forms.scss          |   4 +
 app/mailers/user_mailer.rb                         |  33 ++++++
 app/models/form/challenge.rb                       |   8 ++
 app/models/user.rb                                 |   9 +-
 app/views/auth/challenges/new.html.haml            |  15 +++
 app/views/auth/shared/_links.html.haml             |   2 +-
 .../two_factor_authentications/show.html.haml      |  38 +++----
 .../user_mailer/two_factor_disabled.html.haml      |  43 ++++++++
 app/views/user_mailer/two_factor_disabled.text.erb |   7 ++
 app/views/user_mailer/two_factor_enabled.html.haml |  43 ++++++++
 app/views/user_mailer/two_factor_enabled.text.erb  |   7 ++
 .../two_factor_recovery_codes_changed.html.haml    |  43 ++++++++
 .../two_factor_recovery_codes_changed.text.erb     |   7 ++
 config/locales/devise.en.yml                       |  12 +++
 config/locales/en.yml                              |   5 +
 config/locales/simple_form.en.yml                  |   2 +
 config/routes.rb                                   |   1 +
 .../controllers/auth/challenges_controller_spec.rb |  46 +++++++++
 spec/controllers/auth/sessions_controller_spec.rb  |   2 +-
 .../concerns/challengable_concern_spec.rb          | 114 +++++++++++++++++++++
 .../confirmations_controller_spec.rb               |  10 +-
 .../recovery_codes_controller_spec.rb              |   2 +-
 .../two_factor_authentications_controller_spec.rb  |   2 +-
 spec/mailers/previews/user_mailer_preview.rb       |  15 +++
 32 files changed, 567 insertions(+), 50 deletions(-)
 create mode 100644 app/controllers/auth/challenges_controller.rb
 create mode 100644 app/controllers/concerns/challengable_concern.rb
 create mode 100644 app/models/form/challenge.rb
 create mode 100644 app/views/auth/challenges/new.html.haml
 create mode 100644 app/views/user_mailer/two_factor_disabled.html.haml
 create mode 100644 app/views/user_mailer/two_factor_disabled.text.erb
 create mode 100644 app/views/user_mailer/two_factor_enabled.html.haml
 create mode 100644 app/views/user_mailer/two_factor_enabled.text.erb
 create mode 100644 app/views/user_mailer/two_factor_recovery_codes_changed.html.haml
 create mode 100644 app/views/user_mailer/two_factor_recovery_codes_changed.text.erb
 create mode 100644 spec/controllers/auth/challenges_controller_spec.rb
 create mode 100644 spec/controllers/concerns/challengable_concern_spec.rb

(limited to 'spec')

diff --git a/app/controllers/admin/two_factor_authentications_controller.rb b/app/controllers/admin/two_factor_authentications_controller.rb
index 2577a4b17..0652c3a7a 100644
--- a/app/controllers/admin/two_factor_authentications_controller.rb
+++ b/app/controllers/admin/two_factor_authentications_controller.rb
@@ -8,6 +8,7 @@ module Admin
       authorize @user, :disable_2fa?
       @user.disable_two_factor!
       log_action :disable_2fa, @user
+      UserMailer.two_factor_disabled(@user).deliver_later!
       redirect_to admin_accounts_path
     end
 
diff --git a/app/controllers/auth/challenges_controller.rb b/app/controllers/auth/challenges_controller.rb
new file mode 100644
index 000000000..060944240
--- /dev/null
+++ b/app/controllers/auth/challenges_controller.rb
@@ -0,0 +1,22 @@
+# frozen_string_literal: true
+
+class Auth::ChallengesController < ApplicationController
+  include ChallengableConcern
+
+  layout 'auth'
+
+  before_action :authenticate_user!
+
+  skip_before_action :require_functional!
+
+  def create
+    if challenge_passed?
+      session[:challenge_passed_at] = Time.now.utc
+      redirect_to challenge_params[:return_to]
+    else
+      @challenge = Form::Challenge.new(return_to: challenge_params[:return_to])
+      flash.now[:alert] = I18n.t('challenge.invalid_password')
+      render_challenge
+    end
+  end
+end
diff --git a/app/controllers/auth/sessions_controller.rb b/app/controllers/auth/sessions_controller.rb
index 3e93b2e68..b3113bbef 100644
--- a/app/controllers/auth/sessions_controller.rb
+++ b/app/controllers/auth/sessions_controller.rb
@@ -42,6 +42,7 @@ class Auth::SessionsController < Devise::SessionsController
   def destroy
     tmp_stored_location = stored_location_for(:user)
     super
+    session.delete(:challenge_passed_at)
     flash.delete(:notice)
     store_location_for(:user, tmp_stored_location) if continue_after?
   end
diff --git a/app/controllers/concerns/challengable_concern.rb b/app/controllers/concerns/challengable_concern.rb
new file mode 100644
index 000000000..b29d90b3c
--- /dev/null
+++ b/app/controllers/concerns/challengable_concern.rb
@@ -0,0 +1,65 @@
+# frozen_string_literal: true
+
+# This concern is inspired by "sudo mode" on GitHub. It
+# is a way to re-authenticate a user before allowing them
+# to see or perform an action.
+#
+# Add `before_action :require_challenge!` to actions you
+# want to protect.
+#
+# The user will be shown a page to enter the challenge (which
+# is either the password, or just the username when no
+# password exists). Upon passing, there is a grace period
+# during which no challenge will be asked from the user.
+#
+# Accessing challenge-protected resources during the grace
+# period will refresh the grace period.
+module ChallengableConcern
+  extend ActiveSupport::Concern
+
+  CHALLENGE_TIMEOUT = 1.hour.freeze
+
+  def require_challenge!
+    return if skip_challenge?
+
+    if challenge_passed_recently?
+      session[:challenge_passed_at] = Time.now.utc
+      return
+    end
+
+    @challenge = Form::Challenge.new(return_to: request.url)
+
+    if params.key?(:form_challenge)
+      if challenge_passed?
+        session[:challenge_passed_at] = Time.now.utc
+        return
+      else
+        flash.now[:alert] = I18n.t('challenge.invalid_password')
+        render_challenge
+      end
+    else
+      render_challenge
+    end
+  end
+
+  def render_challenge
+    @body_classes = 'lighter'
+    render template: 'auth/challenges/new', layout: 'auth'
+  end
+
+  def challenge_passed?
+    current_user.valid_password?(challenge_params[:current_password])
+  end
+
+  def skip_challenge?
+    current_user.encrypted_password.blank?
+  end
+
+  def challenge_passed_recently?
+    session[:challenge_passed_at].present? && session[:challenge_passed_at] >= CHALLENGE_TIMEOUT.ago
+  end
+
+  def challenge_params
+    params.require(:form_challenge).permit(:current_password, :return_to)
+  end
+end
diff --git a/app/controllers/settings/two_factor_authentication/confirmations_controller.rb b/app/controllers/settings/two_factor_authentication/confirmations_controller.rb
index 46c90bf74..ef4df3339 100644
--- a/app/controllers/settings/two_factor_authentication/confirmations_controller.rb
+++ b/app/controllers/settings/two_factor_authentication/confirmations_controller.rb
@@ -3,9 +3,12 @@
 module Settings
   module TwoFactorAuthentication
     class ConfirmationsController < BaseController
+      include ChallengableConcern
+
       layout 'admin'
 
       before_action :authenticate_user!
+      before_action :require_challenge!
       before_action :ensure_otp_secret
 
       skip_before_action :require_functional!
@@ -22,6 +25,8 @@ module Settings
           @recovery_codes = current_user.generate_otp_backup_codes!
           current_user.save!
 
+          UserMailer.two_factor_enabled(current_user).deliver_later!
+
           render 'settings/two_factor_authentication/recovery_codes/index'
         else
           flash.now[:alert] = I18n.t('two_factor_authentication.wrong_code')
diff --git a/app/controllers/settings/two_factor_authentication/recovery_codes_controller.rb b/app/controllers/settings/two_factor_authentication/recovery_codes_controller.rb
index 09a759860..0c4f5bff7 100644
--- a/app/controllers/settings/two_factor_authentication/recovery_codes_controller.rb
+++ b/app/controllers/settings/two_factor_authentication/recovery_codes_controller.rb
@@ -3,16 +3,22 @@
 module Settings
   module TwoFactorAuthentication
     class RecoveryCodesController < BaseController
+      include ChallengableConcern
+
       layout 'admin'
 
       before_action :authenticate_user!
+      before_action :require_challenge!, on: :create
 
       skip_before_action :require_functional!
 
       def create
         @recovery_codes = current_user.generate_otp_backup_codes!
         current_user.save!
+
+        UserMailer.two_factor_recovery_codes_changed(current_user).deliver_later!
         flash.now[:notice] = I18n.t('two_factor_authentication.recovery_codes_regenerated')
+
         render :index
       end
     end
diff --git a/app/controllers/settings/two_factor_authentications_controller.rb b/app/controllers/settings/two_factor_authentications_controller.rb
index c93b17577..9118a7933 100644
--- a/app/controllers/settings/two_factor_authentications_controller.rb
+++ b/app/controllers/settings/two_factor_authentications_controller.rb
@@ -2,10 +2,13 @@
 
 module Settings
   class TwoFactorAuthenticationsController < BaseController
+    include ChallengableConcern
+
     layout 'admin'
 
     before_action :authenticate_user!
     before_action :verify_otp_required, only: [:create]
+    before_action :require_challenge!, only: [:create]
 
     skip_before_action :require_functional!
 
@@ -23,6 +26,7 @@ module Settings
       if acceptable_code?
         current_user.otp_required_for_login = false
         current_user.save!
+        UserMailer.two_factor_disabled(current_user).deliver_later!
         redirect_to settings_two_factor_authentication_path
       else
         flash.now[:alert] = I18n.t('two_factor_authentication.wrong_code')
diff --git a/app/javascript/styles/mastodon/admin.scss b/app/javascript/styles/mastodon/admin.scss
index 5d4fe4ef8..074eee2cd 100644
--- a/app/javascript/styles/mastodon/admin.scss
+++ b/app/javascript/styles/mastodon/admin.scss
@@ -233,32 +233,35 @@ hr.spacer {
   height: 1px;
 }
 
-.muted-hint {
-  color: $darker-text-color;
+body,
+.admin-wrapper .content {
+  .muted-hint {
+    color: $darker-text-color;
 
-  a {
-    color: $highlight-text-color;
+    a {
+      color: $highlight-text-color;
+    }
   }
-}
 
-.positive-hint {
-  color: $valid-value-color;
-  font-weight: 500;
-}
+  .positive-hint {
+    color: $valid-value-color;
+    font-weight: 500;
+  }
 
-.negative-hint {
-  color: $error-value-color;
-  font-weight: 500;
-}
+  .negative-hint {
+    color: $error-value-color;
+    font-weight: 500;
+  }
 
-.neutral-hint {
-  color: $dark-text-color;
-  font-weight: 500;
-}
+  .neutral-hint {
+    color: $dark-text-color;
+    font-weight: 500;
+  }
 
-.warning-hint {
-  color: $gold-star;
-  font-weight: 500;
+  .warning-hint {
+    color: $gold-star;
+    font-weight: 500;
+  }
 }
 
 .filters {
diff --git a/app/javascript/styles/mastodon/forms.scss b/app/javascript/styles/mastodon/forms.scss
index 16352340b..80ef8797d 100644
--- a/app/javascript/styles/mastodon/forms.scss
+++ b/app/javascript/styles/mastodon/forms.scss
@@ -254,6 +254,10 @@ code {
       &-6 {
         max-width: 50%;
       }
+
+      .actions {
+        margin-top: 27px;
+      }
     }
 
     .fields-group:last-child,
diff --git a/app/mailers/user_mailer.rb b/app/mailers/user_mailer.rb
index b41004acc..6b81f6873 100644
--- a/app/mailers/user_mailer.rb
+++ b/app/mailers/user_mailer.rb
@@ -57,6 +57,39 @@ class UserMailer < Devise::Mailer
     end
   end
 
+  def two_factor_enabled(user, **)
+    @resource = user
+    @instance = Rails.configuration.x.local_domain
+
+    return if @resource.disabled?
+
+    I18n.with_locale(@resource.locale || I18n.default_locale) do
+      mail to: @resource.email, subject: I18n.t('devise.mailer.two_factor_enabled.subject')
+    end
+  end
+
+  def two_factor_disabled(user, **)
+    @resource = user
+    @instance = Rails.configuration.x.local_domain
+
+    return if @resource.disabled?
+
+    I18n.with_locale(@resource.locale || I18n.default_locale) do
+      mail to: @resource.email, subject: I18n.t('devise.mailer.two_factor_disabled.subject')
+    end
+  end
+
+  def two_factor_recovery_codes_changed(user, **)
+    @resource = user
+    @instance = Rails.configuration.x.local_domain
+
+    return if @resource.disabled?
+
+    I18n.with_locale(@resource.locale || I18n.default_locale) do
+      mail to: @resource.email, subject: I18n.t('devise.mailer.two_factor_recovery_codes_changed.subject')
+    end
+  end
+
   def welcome(user)
     @resource = user
     @instance = Rails.configuration.x.local_domain
diff --git a/app/models/form/challenge.rb b/app/models/form/challenge.rb
new file mode 100644
index 000000000..40c99649c
--- /dev/null
+++ b/app/models/form/challenge.rb
@@ -0,0 +1,8 @@
+# frozen_string_literal: true
+
+class Form::Challenge
+  include ActiveModel::Model
+
+  attr_accessor :current_password, :current_username,
+                :return_to
+end
diff --git a/app/models/user.rb b/app/models/user.rb
index 78b82a68f..b48455802 100644
--- a/app/models/user.rb
+++ b/app/models/user.rb
@@ -264,17 +264,20 @@ class User < ApplicationRecord
   end
 
   def password_required?
-    return false if Devise.pam_authentication || Devise.ldap_authentication
+    return false if external?
+
     super
   end
 
   def send_reset_password_instructions
-    return false if encrypted_password.blank? && (Devise.pam_authentication || Devise.ldap_authentication)
+    return false if encrypted_password.blank?
+
     super
   end
 
   def reset_password!(new_password, new_password_confirmation)
-    return false if encrypted_password.blank? && (Devise.pam_authentication || Devise.ldap_authentication)
+    return false if encrypted_password.blank?
+
     super
   end
 
diff --git a/app/views/auth/challenges/new.html.haml b/app/views/auth/challenges/new.html.haml
new file mode 100644
index 000000000..9aef2c35d
--- /dev/null
+++ b/app/views/auth/challenges/new.html.haml
@@ -0,0 +1,15 @@
+- content_for :page_title do
+  = t('challenge.prompt')
+
+= simple_form_for @challenge, url: request.get? ? auth_challenge_path : '' do |f|
+  = f.input :return_to, as: :hidden
+
+  .field-group
+    = f.input :current_password, wrapper: :with_block_label, input_html: { :autocomplete => 'off', :autofocus => true }, label: t('challenge.prompt'), required: true
+
+  .actions
+    = f.button :button, t('challenge.confirm'), type: :submit
+
+  %p.hint.subtle-hint= t('challenge.hint_html')
+
+.form-footer= render 'auth/shared/links'
diff --git a/app/views/auth/shared/_links.html.haml b/app/views/auth/shared/_links.html.haml
index e6c3f7cca..66ed5b93f 100644
--- a/app/views/auth/shared/_links.html.haml
+++ b/app/views/auth/shared/_links.html.haml
@@ -11,7 +11,7 @@
     - if controller_name != 'passwords' && controller_name != 'registrations'
       %li= link_to t('auth.forgot_password'), new_user_password_path
 
-  - if controller_name != 'confirmations'
+  - if controller_name != 'confirmations' && (!user_signed_in? || !current_user.confirmed? || current_user.unconfirmed_email.present?)
     %li= link_to t('auth.didnt_get_confirmation'), new_user_confirmation_path
 
   - if user_signed_in? && controller_name != 'setup'
diff --git a/app/views/settings/two_factor_authentications/show.html.haml b/app/views/settings/two_factor_authentications/show.html.haml
index 93509e022..f1eecd000 100644
--- a/app/views/settings/two_factor_authentications/show.html.haml
+++ b/app/views/settings/two_factor_authentications/show.html.haml
@@ -2,33 +2,35 @@
   = t('settings.two_factor_authentication')
 
 - if current_user.otp_required_for_login
-  %p.positive-hint
-    = fa_icon 'check'
-    = ' '
-    = t 'two_factor_authentication.enabled'
+  %p.hint
+    %span.positive-hint
+      = fa_icon 'check'
+      = ' '
+      = t 'two_factor_authentication.enabled'
 
-  %hr/
+  %hr.spacer/
 
   = simple_form_for @confirmation, url: settings_two_factor_authentication_path, method: :delete do |f|
-    = f.input :otp_attempt, wrapper: :with_label, hint: t('two_factor_authentication.code_hint'), label: t('simple_form.labels.defaults.otp_attempt'), input_html: { :autocomplete => 'off' }, required: true
+    .fields-group
+      = f.input :otp_attempt, wrapper: :with_block_label, hint: t('two_factor_authentication.code_hint'), label: t('simple_form.labels.defaults.otp_attempt'), input_html: { :autocomplete => 'off' }, required: true
 
     .actions
-      = f.button :button, t('two_factor_authentication.disable'), type: :submit
+      = f.button :button, t('two_factor_authentication.disable'), type: :submit, class: 'negative'
 
-  %hr/
+  %hr.spacer/
 
-  %h6= t('two_factor_authentication.recovery_codes')
-  %p.muted-hint
-    = t('two_factor_authentication.lost_recovery_codes')
-    = link_to t('two_factor_authentication.generate_recovery_codes'),
-      settings_two_factor_authentication_recovery_codes_path,
-      data: { method: :post }
+  %h3= t('two_factor_authentication.recovery_codes')
+  %p.muted-hint= t('two_factor_authentication.lost_recovery_codes')
+
+  %hr.spacer/
+
+  .simple_form
+    = link_to t('two_factor_authentication.generate_recovery_codes'), settings_two_factor_authentication_recovery_codes_path, data: { method: :post }, class: 'block-button'
 
 - else
   .simple_form
     %p.hint= t('two_factor_authentication.description_html')
 
-    = link_to t('two_factor_authentication.setup'),
-      settings_two_factor_authentication_path,
-      data: { method: :post },
-      class: 'block-button'
+    %hr.spacer/
+
+    = link_to t('two_factor_authentication.setup'), settings_two_factor_authentication_path, data: { method: :post }, class: 'block-button'
diff --git a/app/views/user_mailer/two_factor_disabled.html.haml b/app/views/user_mailer/two_factor_disabled.html.haml
new file mode 100644
index 000000000..651c6f940
--- /dev/null
+++ b/app/views/user_mailer/two_factor_disabled.html.haml
@@ -0,0 +1,43 @@
+%table.email-table{ cellspacing: 0, cellpadding: 0 }
+  %tbody
+    %tr
+      %td.email-body
+        .email-container
+          %table.content-section{ cellspacing: 0, cellpadding: 0 }
+            %tbody
+              %tr
+                %td.content-cell.hero
+                  .email-row
+                    .col-6
+                      %table.column{ cellspacing: 0, cellpadding: 0 }
+                        %tbody
+                          %tr
+                            %td.column-cell.text-center.padded
+                              %table.hero-icon.alert-icon{ align: 'center', cellspacing: 0, cellpadding: 0 }
+                                %tbody
+                                  %tr
+                                    %td
+                                      = image_tag full_pack_url('media/images/mailer/icon_lock_open.png'), alt: ''
+
+                              %h1= t 'devise.mailer.two_factor_disabled.title'
+                              %p.lead= t 'devise.mailer.two_factor_disabled.explanation'
+
+%table.email-table{ cellspacing: 0, cellpadding: 0 }
+  %tbody
+    %tr
+      %td.email-body
+        .email-container
+          %table.content-section{ cellspacing: 0, cellpadding: 0 }
+            %tbody
+              %tr
+                %td.content-cell.content-start
+                  %table.column{ cellspacing: 0, cellpadding: 0 }
+                    %tbody
+                      %tr
+                        %td.column-cell.button-cell
+                          %table.button{ align: 'center', cellspacing: 0, cellpadding: 0 }
+                            %tbody
+                              %tr
+                                %td.button-primary
+                                  = link_to edit_user_registration_url do
+                                    %span= t('settings.account_settings')
diff --git a/app/views/user_mailer/two_factor_disabled.text.erb b/app/views/user_mailer/two_factor_disabled.text.erb
new file mode 100644
index 000000000..73be1ddc2
--- /dev/null
+++ b/app/views/user_mailer/two_factor_disabled.text.erb
@@ -0,0 +1,7 @@
+<%= t 'devise.mailer.two_factor_disabled.title' %>
+
+===
+
+<%= t 'devise.mailer.two_factor_disabled.explanation' %>
+
+=> <%= edit_user_registration_url %>
diff --git a/app/views/user_mailer/two_factor_enabled.html.haml b/app/views/user_mailer/two_factor_enabled.html.haml
new file mode 100644
index 000000000..fc31bd979
--- /dev/null
+++ b/app/views/user_mailer/two_factor_enabled.html.haml
@@ -0,0 +1,43 @@
+%table.email-table{ cellspacing: 0, cellpadding: 0 }
+  %tbody
+    %tr
+      %td.email-body
+        .email-container
+          %table.content-section{ cellspacing: 0, cellpadding: 0 }
+            %tbody
+              %tr
+                %td.content-cell.hero
+                  .email-row
+                    .col-6
+                      %table.column{ cellspacing: 0, cellpadding: 0 }
+                        %tbody
+                          %tr
+                            %td.column-cell.text-center.padded
+                              %table.hero-icon{ align: 'center', cellspacing: 0, cellpadding: 0 }
+                                %tbody
+                                  %tr
+                                    %td
+                                      = image_tag full_pack_url('media/images/mailer/icon_lock_open.png'), alt: ''
+
+                              %h1= t 'devise.mailer.two_factor_enabled.title'
+                              %p.lead= t 'devise.mailer.two_factor_enabled.explanation'
+
+%table.email-table{ cellspacing: 0, cellpadding: 0 }
+  %tbody
+    %tr
+      %td.email-body
+        .email-container
+          %table.content-section{ cellspacing: 0, cellpadding: 0 }
+            %tbody
+              %tr
+                %td.content-cell.content-start
+                  %table.column{ cellspacing: 0, cellpadding: 0 }
+                    %tbody
+                      %tr
+                        %td.column-cell.button-cell
+                          %table.button{ align: 'center', cellspacing: 0, cellpadding: 0 }
+                            %tbody
+                              %tr
+                                %td.button-primary
+                                  = link_to edit_user_registration_url do
+                                    %span= t('settings.account_settings')
diff --git a/app/views/user_mailer/two_factor_enabled.text.erb b/app/views/user_mailer/two_factor_enabled.text.erb
new file mode 100644
index 000000000..4319dddbf
--- /dev/null
+++ b/app/views/user_mailer/two_factor_enabled.text.erb
@@ -0,0 +1,7 @@
+<%= t 'devise.mailer.two_factor_enabled.title' %>
+
+===
+
+<%= t 'devise.mailer.two_factor_enabled.explanation' %>
+
+=> <%= edit_user_registration_url %>
diff --git a/app/views/user_mailer/two_factor_recovery_codes_changed.html.haml b/app/views/user_mailer/two_factor_recovery_codes_changed.html.haml
new file mode 100644
index 000000000..833708868
--- /dev/null
+++ b/app/views/user_mailer/two_factor_recovery_codes_changed.html.haml
@@ -0,0 +1,43 @@
+%table.email-table{ cellspacing: 0, cellpadding: 0 }
+  %tbody
+    %tr
+      %td.email-body
+        .email-container
+          %table.content-section{ cellspacing: 0, cellpadding: 0 }
+            %tbody
+              %tr
+                %td.content-cell.hero
+                  .email-row
+                    .col-6
+                      %table.column{ cellspacing: 0, cellpadding: 0 }
+                        %tbody
+                          %tr
+                            %td.column-cell.text-center.padded
+                              %table.hero-icon.alert-icon{ align: 'center', cellspacing: 0, cellpadding: 0 }
+                                %tbody
+                                  %tr
+                                    %td
+                                      = image_tag full_pack_url('media/images/mailer/icon_lock_open.png'), alt: ''
+
+                              %h1= t 'devise.mailer.two_factor_recovery_codes_changed.title'
+                              %p.lead= t 'devise.mailer.two_factor_recovery_codes_changed.explanation'
+
+%table.email-table{ cellspacing: 0, cellpadding: 0 }
+  %tbody
+    %tr
+      %td.email-body
+        .email-container
+          %table.content-section{ cellspacing: 0, cellpadding: 0 }
+            %tbody
+              %tr
+                %td.content-cell.content-start
+                  %table.column{ cellspacing: 0, cellpadding: 0 }
+                    %tbody
+                      %tr
+                        %td.column-cell.button-cell
+                          %table.button{ align: 'center', cellspacing: 0, cellpadding: 0 }
+                            %tbody
+                              %tr
+                                %td.button-primary
+                                  = link_to edit_user_registration_url do
+                                    %span= t('settings.account_settings')
diff --git a/app/views/user_mailer/two_factor_recovery_codes_changed.text.erb b/app/views/user_mailer/two_factor_recovery_codes_changed.text.erb
new file mode 100644
index 000000000..6ed12fc08
--- /dev/null
+++ b/app/views/user_mailer/two_factor_recovery_codes_changed.text.erb
@@ -0,0 +1,7 @@
+<%= t 'devise.mailer.two_factor_recovery_codes_changed.title' %>
+
+===
+
+<%= t 'devise.mailer.two_factor_recovery_codes_changed.explanation' %>
+
+=> <%= edit_user_registration_url %>
diff --git a/config/locales/devise.en.yml b/config/locales/devise.en.yml
index 5defa6624..726d2426a 100644
--- a/config/locales/devise.en.yml
+++ b/config/locales/devise.en.yml
@@ -46,6 +46,18 @@ en:
         extra: If you didn't request this, please ignore this email. Your password won't change until you access the link above and create a new one.
         subject: 'Mastodon: Reset password instructions'
         title: Password reset
+      two_factor_disabled:
+        explanation: Two-factor authentication for your account has been disabled. Login is now possible using only e-mail address and password.
+        subject: 'Mastodon: Two-factor authentication disabled'
+        title: 2FA disabled
+      two_factor_enabled:
+        explanation: Two-factor authentication has been enabled for your account. A token generated by the paired TOTP app will be required for login.
+        subject: 'Mastodon: Two-factor authentication enabled'
+        title: 2FA enabled
+      two_factor_recovery_codes_changed:
+        explanation: The previous recovery codes have been invalidated and new ones generated.
+        subject: 'Mastodon: Two-factor recovery codes re-generated'
+        title: 2FA recovery codes changed
       unlock_instructions:
         subject: 'Mastodon: Unlock instructions'
     omniauth_callbacks:
diff --git a/config/locales/en.yml b/config/locales/en.yml
index f05fdd48b..da06b0e51 100644
--- a/config/locales/en.yml
+++ b/config/locales/en.yml
@@ -621,6 +621,11 @@ en:
       return: Show the user's profile
       web: Go to web
     title: Follow %{acct}
+  challenge:
+    confirm: Continue
+    hint_html: "<strong>Tip:</strong> We won't ask you for your password again for the next hour."
+    invalid_password: Invalid password
+    prompt: Confirm password to continue
   datetime:
     distance_in_words:
       about_x_hours: "%{count}h"
diff --git a/config/locales/simple_form.en.yml b/config/locales/simple_form.en.yml
index c542377a9..c9ffcfc13 100644
--- a/config/locales/simple_form.en.yml
+++ b/config/locales/simple_form.en.yml
@@ -43,6 +43,8 @@ en:
         domain: This domain will be able to fetch data from this server and incoming data from it will be processed and stored
       featured_tag:
         name: 'You might want to use one of these:'
+      form_challenge:
+        current_password: You are entering a secure area
       imports:
         data: CSV file exported from another Mastodon server
       invite_request:
diff --git a/config/routes.rb b/config/routes.rb
index a4dee2842..9ad1ea65d 100644
--- a/config/routes.rb
+++ b/config/routes.rb
@@ -41,6 +41,7 @@ Rails.application.routes.draw do
 
     namespace :auth do
       resource :setup, only: [:show, :update], controller: :setup
+      resource :challenge, only: [:create], controller: :challenges
     end
   end
 
diff --git a/spec/controllers/auth/challenges_controller_spec.rb b/spec/controllers/auth/challenges_controller_spec.rb
new file mode 100644
index 000000000..2a6ca301e
--- /dev/null
+++ b/spec/controllers/auth/challenges_controller_spec.rb
@@ -0,0 +1,46 @@
+# frozen_string_literal: true
+
+require 'rails_helper'
+
+describe Auth::ChallengesController, type: :controller do
+  render_views
+
+  let(:password) { 'foobar12345' }
+  let(:user) { Fabricate(:user, password: password) }
+
+  before do
+    sign_in user
+  end
+
+  describe 'POST #create' do
+    let(:return_to) { edit_user_registration_path }
+
+    context 'with correct password' do
+      before { post :create, params: { form_challenge: { return_to: return_to, current_password: password } } }
+
+      it 'redirects back' do
+        expect(response).to redirect_to(return_to)
+      end
+
+      it 'sets session' do
+        expect(session[:challenge_passed_at]).to_not be_nil
+      end
+    end
+
+    context 'with incorrect password' do
+      before { post :create, params: { form_challenge: { return_to: return_to, current_password: 'hhfggjjd562' } } }
+
+      it 'renders challenge' do
+        expect(response).to render_template('auth/challenges/new')
+      end
+
+      it 'displays error' do
+        expect(response.body).to include 'Invalid password'
+      end
+
+      it 'does not set session' do
+        expect(session[:challenge_passed_at]).to be_nil
+      end
+    end
+  end
+end
diff --git a/spec/controllers/auth/sessions_controller_spec.rb b/spec/controllers/auth/sessions_controller_spec.rb
index 7ed5edde0..1950c173a 100644
--- a/spec/controllers/auth/sessions_controller_spec.rb
+++ b/spec/controllers/auth/sessions_controller_spec.rb
@@ -80,7 +80,7 @@ RSpec.describe Auth::SessionsController, type: :controller do
         let(:user) do
           account = Fabricate.build(:account, username: 'pam_user1')
           account.save!(validate: false)
-          user = Fabricate(:user, email: 'pam@example.com', password: nil, account: account)
+          user = Fabricate(:user, email: 'pam@example.com', password: nil, account: account, external: true)
           user
         end
 
diff --git a/spec/controllers/concerns/challengable_concern_spec.rb b/spec/controllers/concerns/challengable_concern_spec.rb
new file mode 100644
index 000000000..4db3b740d
--- /dev/null
+++ b/spec/controllers/concerns/challengable_concern_spec.rb
@@ -0,0 +1,114 @@
+# frozen_string_literal: true
+
+require 'rails_helper'
+
+RSpec.describe ChallengableConcern, type: :controller do
+  controller(ApplicationController) do
+    include ChallengableConcern
+
+    before_action :require_challenge!
+
+    def foo
+      render plain: 'foo'
+    end
+
+    def bar
+      render plain: 'bar'
+    end
+  end
+
+  before do
+    routes.draw do
+      get  'foo' => 'anonymous#foo'
+      post 'bar' => 'anonymous#bar'
+    end
+  end
+
+  context 'with a no-password user' do
+    let(:user) { Fabricate(:user, external: true, password: nil) }
+
+    before do
+      sign_in user
+    end
+
+    context 'for GET requests' do
+      before { get :foo }
+
+      it 'does not ask for password' do
+        expect(response.body).to eq 'foo'
+      end
+    end
+
+    context 'for POST requests' do
+      before { post :bar }
+
+      it 'does not ask for password' do
+        expect(response.body).to eq 'bar'
+      end
+    end
+  end
+
+  context 'with recent challenge in session' do
+    let(:password) { 'foobar12345' }
+    let(:user) { Fabricate(:user, password: password) }
+
+    before do
+      sign_in user
+    end
+
+    context 'for GET requests' do
+      before { get :foo, session: { challenge_passed_at: Time.now.utc } }
+
+      it 'does not ask for password' do
+        expect(response.body).to eq 'foo'
+      end
+    end
+
+    context 'for POST requests' do
+      before { post :bar, session: { challenge_passed_at: Time.now.utc } }
+
+      it 'does not ask for password' do
+        expect(response.body).to eq 'bar'
+      end
+    end
+  end
+
+  context 'with a password user' do
+    let(:password) { 'foobar12345' }
+    let(:user) { Fabricate(:user, password: password) }
+
+    before do
+      sign_in user
+    end
+
+    context 'for GET requests' do
+      before { get :foo }
+
+      it 'renders challenge' do
+        expect(response).to render_template('auth/challenges/new')
+      end
+
+      # See Auth::ChallengesControllerSpec
+    end
+
+    context 'for POST requests' do
+      before { post :bar }
+
+      it 'renders challenge' do
+        expect(response).to render_template('auth/challenges/new')
+      end
+
+      it 'accepts correct password' do
+        post :bar, params: { form_challenge: { current_password: password } }
+        expect(response.body).to eq 'bar'
+        expect(session[:challenge_passed_at]).to_not be_nil
+      end
+
+      it 'rejects wrong password' do
+        post :bar, params: { form_challenge: { current_password: 'dddfff888123' } }
+        expect(response.body).to render_template('auth/challenges/new')
+        expect(session[:challenge_passed_at]).to be_nil
+      end
+    end
+  end
+end
diff --git a/spec/controllers/settings/two_factor_authentication/confirmations_controller_spec.rb b/spec/controllers/settings/two_factor_authentication/confirmations_controller_spec.rb
index 2e5a9325c..336f13127 100644
--- a/spec/controllers/settings/two_factor_authentication/confirmations_controller_spec.rb
+++ b/spec/controllers/settings/two_factor_authentication/confirmations_controller_spec.rb
@@ -24,7 +24,7 @@ describe Settings::TwoFactorAuthentication::ConfirmationsController do
     context 'when signed in' do
       subject do
         sign_in user, scope: :user
-        get :new
+        get :new, session: { challenge_passed_at: Time.now.utc }
       end
 
       include_examples 'renders :new'
@@ -37,7 +37,7 @@ describe Settings::TwoFactorAuthentication::ConfirmationsController do
 
     it 'redirects if user do not have otp_secret' do
       sign_in user_without_otp_secret, scope: :user
-      get :new
+      get :new, session: { challenge_passed_at: Time.now.utc }
       expect(response).to redirect_to('/settings/two_factor_authentication')
     end
   end
@@ -50,7 +50,7 @@ describe Settings::TwoFactorAuthentication::ConfirmationsController do
 
       describe 'when form_two_factor_confirmation parameter is not provided' do
         it 'raises ActionController::ParameterMissing' do
-          post :create, params: {}
+          post :create, params: {}, session: { challenge_passed_at: Time.now.utc }
           expect(response).to have_http_status(400)
         end
       end
@@ -68,7 +68,7 @@ describe Settings::TwoFactorAuthentication::ConfirmationsController do
             true
           end
 
-          post :create, params: { form_two_factor_confirmation: { otp_attempt: '123456' } }
+          post :create, params: { form_two_factor_confirmation: { otp_attempt: '123456' } }, session: { challenge_passed_at: Time.now.utc }
 
           expect(assigns(:recovery_codes)).to eq otp_backup_codes
           expect(flash[:notice]).to eq 'Two-factor authentication successfully enabled'
@@ -85,7 +85,7 @@ describe Settings::TwoFactorAuthentication::ConfirmationsController do
             false
           end
 
-          post :create, params: { form_two_factor_confirmation: { otp_attempt: '123456' } }
+          post :create, params: { form_two_factor_confirmation: { otp_attempt: '123456' } }, session: { challenge_passed_at: Time.now.utc }
         end
 
         it 'renders the new view' do
diff --git a/spec/controllers/settings/two_factor_authentication/recovery_codes_controller_spec.rb b/spec/controllers/settings/two_factor_authentication/recovery_codes_controller_spec.rb
index c04760e53..630cec428 100644
--- a/spec/controllers/settings/two_factor_authentication/recovery_codes_controller_spec.rb
+++ b/spec/controllers/settings/two_factor_authentication/recovery_codes_controller_spec.rb
@@ -15,7 +15,7 @@ describe Settings::TwoFactorAuthentication::RecoveryCodesController do
       end
 
       sign_in user, scope: :user
-      post :create
+      post :create, session: { challenge_passed_at: Time.now.utc }
 
       expect(assigns(:recovery_codes)).to eq otp_backup_codes
       expect(flash[:notice]).to eq 'Recovery codes successfully regenerated'
diff --git a/spec/controllers/settings/two_factor_authentications_controller_spec.rb b/spec/controllers/settings/two_factor_authentications_controller_spec.rb
index 922231ded..9df9763fd 100644
--- a/spec/controllers/settings/two_factor_authentications_controller_spec.rb
+++ b/spec/controllers/settings/two_factor_authentications_controller_spec.rb
@@ -58,7 +58,7 @@ describe Settings::TwoFactorAuthenticationsController do
       describe 'when creation succeeds' do
         it 'updates user secret' do
           before = user.otp_secret
-          post :create
+          post :create, session: { challenge_passed_at: Time.now.utc }
 
           expect(user.reload.otp_secret).not_to eq(before)
           expect(response).to redirect_to(new_settings_two_factor_authentication_confirmation_path)
diff --git a/spec/mailers/previews/user_mailer_preview.rb b/spec/mailers/previews/user_mailer_preview.rb
index ead3b3baa..464f177d0 100644
--- a/spec/mailers/previews/user_mailer_preview.rb
+++ b/spec/mailers/previews/user_mailer_preview.rb
@@ -18,6 +18,21 @@ class UserMailerPreview < ActionMailer::Preview
     UserMailer.password_change(User.first)
   end
 
+  # Preview this email at http://localhost:3000/rails/mailers/user_mailer/two_factor_disabled
+  def two_factor_disabled
+    UserMailer.two_factor_disabled(User.first)
+  end
+
+  # Preview this email at http://localhost:3000/rails/mailers/user_mailer/two_factor_enabled
+  def two_factor_enabled
+    UserMailer.two_factor_enabled(User.first)
+  end
+
+  # Preview this email at http://localhost:3000/rails/mailers/user_mailer/two_factor_recovery_codes_changed
+  def two_factor_recovery_codes_changed
+    UserMailer.two_factor_recovery_codes_changed(User.first)
+  end
+
   # Preview this email at http://localhost:3000/rails/mailers/user_mailer/reconfirmation_instructions
   def reconfirmation_instructions
     user = User.first
-- 
cgit 


From 3ed94dcc1acf73f1d0d1ab43567b88ee953f57c9 Mon Sep 17 00:00:00 2001
From: Eugen Rochko <eugen@zeonfederated.com>
Date: Thu, 19 Sep 2019 20:58:19 +0200
Subject: Add account migration UI (#11846)

Fix #10736

- Change data export to be available for non-functional accounts
- Change non-functional accounts to include redirecting accounts
---
 .../concerns/export_controller_concern.rb          |  7 ++
 app/controllers/settings/aliases_controller.rb     | 42 +++++++++++
 app/controllers/settings/exports_controller.rb     |  7 ++
 app/controllers/settings/migrations_controller.rb  | 48 ++++++++++---
 app/helpers/settings_helper.rb                     |  8 +++
 app/models/account_alias.rb                        | 41 +++++++++++
 app/models/account_migration.rb                    | 74 +++++++++++++++++++
 app/models/concerns/account_associations.rb        |  2 +
 app/models/form/migration.rb                       | 25 -------
 app/models/remote_follow.rb                        |  2 +-
 app/models/user.rb                                 |  2 +-
 app/serializers/activitypub/move_serializer.rb     | 26 +++++++
 app/views/auth/registrations/_status.html.haml     | 30 ++++----
 app/views/auth/registrations/edit.html.haml        |  2 +-
 app/views/settings/aliases/index.html.haml         | 29 ++++++++
 app/views/settings/exports/show.html.haml          |  4 ++
 app/views/settings/migrations/show.html.haml       | 84 +++++++++++++++++++---
 app/views/settings/profiles/show.html.haml         |  5 ++
 .../activitypub/move_distribution_worker.rb        | 32 +++++++++
 config/locales/en.yml                              | 38 ++++++++--
 config/locales/simple_form.en.yml                  | 10 +++
 config/navigation.rb                               |  8 +--
 config/routes.rb                                   |  8 ++-
 .../20190914202517_create_account_migrations.rb    | 12 ++++
 .../20190915194355_create_account_aliases.rb       | 11 +++
 db/schema.rb                                       | 23 ++++++
 .../settings/migrations_controller_spec.rb         | 14 ++--
 spec/fabricators/account_alias_fabricator.rb       |  5 ++
 spec/fabricators/account_migration_fabricator.rb   |  6 ++
 spec/models/account_alias_spec.rb                  |  5 ++
 spec/models/account_migration_spec.rb              |  5 ++
 31 files changed, 542 insertions(+), 73 deletions(-)
 create mode 100644 app/controllers/settings/aliases_controller.rb
 create mode 100644 app/models/account_alias.rb
 create mode 100644 app/models/account_migration.rb
 delete mode 100644 app/models/form/migration.rb
 create mode 100644 app/serializers/activitypub/move_serializer.rb
 create mode 100644 app/views/settings/aliases/index.html.haml
 create mode 100644 app/workers/activitypub/move_distribution_worker.rb
 create mode 100644 db/migrate/20190914202517_create_account_migrations.rb
 create mode 100644 db/migrate/20190915194355_create_account_aliases.rb
 create mode 100644 spec/fabricators/account_alias_fabricator.rb
 create mode 100644 spec/fabricators/account_migration_fabricator.rb
 create mode 100644 spec/models/account_alias_spec.rb
 create mode 100644 spec/models/account_migration_spec.rb

(limited to 'spec')

diff --git a/app/controllers/concerns/export_controller_concern.rb b/app/controllers/concerns/export_controller_concern.rb
index e20b71a30..bfe990c82 100644
--- a/app/controllers/concerns/export_controller_concern.rb
+++ b/app/controllers/concerns/export_controller_concern.rb
@@ -5,7 +5,10 @@ module ExportControllerConcern
 
   included do
     before_action :authenticate_user!
+    before_action :require_not_suspended!
     before_action :load_export
+
+    skip_before_action :require_functional!
   end
 
   private
@@ -27,4 +30,8 @@ module ExportControllerConcern
   def export_filename
     "#{controller_name}.csv"
   end
+
+  def require_not_suspended!
+    forbidden if current_account.suspended?
+  end
 end
diff --git a/app/controllers/settings/aliases_controller.rb b/app/controllers/settings/aliases_controller.rb
new file mode 100644
index 000000000..2b675f065
--- /dev/null
+++ b/app/controllers/settings/aliases_controller.rb
@@ -0,0 +1,42 @@
+# frozen_string_literal: true
+
+class Settings::AliasesController < Settings::BaseController
+  layout 'admin'
+
+  before_action :authenticate_user!
+  before_action :set_aliases, except: :destroy
+  before_action :set_alias, only: :destroy
+
+  def index
+    @alias = current_account.aliases.build
+  end
+
+  def create
+    @alias = current_account.aliases.build(resource_params)
+
+    if @alias.save
+      redirect_to settings_aliases_path, notice: I18n.t('aliases.created_msg')
+    else
+      render :show
+    end
+  end
+
+  def destroy
+    @alias.destroy!
+    redirect_to settings_aliases_path, notice: I18n.t('aliases.deleted_msg')
+  end
+
+  private
+
+  def resource_params
+    params.require(:account_alias).permit(:acct)
+  end
+
+  def set_alias
+    @alias = current_account.aliases.find(params[:id])
+  end
+
+  def set_aliases
+    @aliases = current_account.aliases.order(id: :desc).reject(&:new_record?)
+  end
+end
diff --git a/app/controllers/settings/exports_controller.rb b/app/controllers/settings/exports_controller.rb
index 3012fbf77..0e93d07a9 100644
--- a/app/controllers/settings/exports_controller.rb
+++ b/app/controllers/settings/exports_controller.rb
@@ -6,6 +6,9 @@ class Settings::ExportsController < Settings::BaseController
   layout 'admin'
 
   before_action :authenticate_user!
+  before_action :require_not_suspended!
+
+  skip_before_action :require_functional!
 
   def show
     @export  = Export.new(current_account)
@@ -34,4 +37,8 @@ class Settings::ExportsController < Settings::BaseController
   def lock_options
     { redis: Redis.current, key: "backup:#{current_user.id}" }
   end
+
+  def require_not_suspended!
+    forbidden if current_account.suspended?
+  end
 end
diff --git a/app/controllers/settings/migrations_controller.rb b/app/controllers/settings/migrations_controller.rb
index 59eb48779..90092c692 100644
--- a/app/controllers/settings/migrations_controller.rb
+++ b/app/controllers/settings/migrations_controller.rb
@@ -4,31 +4,59 @@ class Settings::MigrationsController < Settings::BaseController
   layout 'admin'
 
   before_action :authenticate_user!
+  before_action :require_not_suspended!
+  before_action :set_migrations
+  before_action :set_cooldown
+
+  skip_before_action :require_functional!
 
   def show
-    @migration = Form::Migration.new(account: current_account.moved_to_account)
+    @migration = current_account.migrations.build
   end
 
-  def update
-    @migration = Form::Migration.new(resource_params)
+  def create
+    @migration = current_account.migrations.build(resource_params)
 
-    if @migration.valid? && migration_account_changed?
-      current_account.update!(moved_to_account: @migration.account)
+    if @migration.save_with_challenge(current_user)
+      current_account.update!(moved_to_account: @migration.target_account)
       ActivityPub::UpdateDistributionWorker.perform_async(current_account.id)
-      redirect_to settings_migration_path, notice: I18n.t('migrations.updated_msg')
+      ActivityPub::MoveDistributionWorker.perform_async(@migration.id)
+      redirect_to settings_migration_path, notice: I18n.t('migrations.moved_msg', acct: current_account.moved_to_account.acct)
     else
       render :show
     end
   end
 
+  def cancel
+    if current_account.moved_to_account_id.present?
+      current_account.update!(moved_to_account: nil)
+      ActivityPub::UpdateDistributionWorker.perform_async(current_account.id)
+    end
+
+    redirect_to settings_migration_path, notice: I18n.t('migrations.cancelled_msg')
+  end
+
+  helper_method :on_cooldown?
+
   private
 
   def resource_params
-    params.require(:migration).permit(:acct)
+    params.require(:account_migration).permit(:acct, :current_password, :current_username)
+  end
+
+  def set_migrations
+    @migrations = current_account.migrations.includes(:target_account).order(id: :desc).reject(&:new_record?)
+  end
+
+  def set_cooldown
+    @cooldown = current_account.migrations.within_cooldown.first
+  end
+
+  def on_cooldown?
+    @cooldown.present?
   end
 
-  def migration_account_changed?
-    current_account.moved_to_account_id != @migration.account&.id &&
-      current_account.id != @migration.account&.id
+  def require_not_suspended!
+    forbidden if current_account.suspended?
   end
 end
diff --git a/app/helpers/settings_helper.rb b/app/helpers/settings_helper.rb
index 2b3fd1263..ecc73baf5 100644
--- a/app/helpers/settings_helper.rb
+++ b/app/helpers/settings_helper.rb
@@ -87,4 +87,12 @@ module SettingsHelper
       'desktop'
     end
   end
+
+  def compact_account_link_to(account)
+    return if account.nil?
+
+    link_to ActivityPub::TagManager.instance.url_for(account), class: 'name-tag', title: account.acct do
+      safe_join([image_tag(account.avatar.url, width: 15, height: 15, alt: display_name(account), class: 'avatar'), content_tag(:span, account.acct, class: 'username')], ' ')
+    end
+  end
 end
diff --git a/app/models/account_alias.rb b/app/models/account_alias.rb
new file mode 100644
index 000000000..e9a0dd79e
--- /dev/null
+++ b/app/models/account_alias.rb
@@ -0,0 +1,41 @@
+# frozen_string_literal: true
+
+# == Schema Information
+#
+# Table name: account_aliases
+#
+#  id         :bigint(8)        not null, primary key
+#  account_id :bigint(8)
+#  acct       :string           default(""), not null
+#  uri        :string           default(""), not null
+#  created_at :datetime         not null
+#  updated_at :datetime         not null
+#
+
+class AccountAlias < ApplicationRecord
+  belongs_to :account
+
+  validates :acct, presence: true, domain: { acct: true }
+  validates :uri, presence: true
+
+  before_validation :set_uri
+  after_create :add_to_account
+  after_destroy :remove_from_account
+
+  private
+
+  def set_uri
+    target_account = ResolveAccountService.new.call(acct)
+    self.uri       = ActivityPub::TagManager.instance.uri_for(target_account) unless target_account.nil?
+  rescue Goldfinger::Error, HTTP::Error, OpenSSL::SSL::SSLError, Mastodon::Error
+    # Validation will take care of it
+  end
+
+  def add_to_account
+    account.update(also_known_as: account.also_known_as + [uri])
+  end
+
+  def remove_from_account
+    account.update(also_known_as: account.also_known_as.reject { |x| x == uri })
+  end
+end
diff --git a/app/models/account_migration.rb b/app/models/account_migration.rb
new file mode 100644
index 000000000..15830bffb
--- /dev/null
+++ b/app/models/account_migration.rb
@@ -0,0 +1,74 @@
+# frozen_string_literal: true
+
+# == Schema Information
+#
+# Table name: account_migrations
+#
+#  id                :bigint(8)        not null, primary key
+#  account_id        :bigint(8)
+#  acct              :string           default(""), not null
+#  followers_count   :bigint(8)        default(0), not null
+#  target_account_id :bigint(8)
+#  created_at        :datetime         not null
+#  updated_at        :datetime         not null
+#
+
+class AccountMigration < ApplicationRecord
+  COOLDOWN_PERIOD = 30.days.freeze
+
+  belongs_to :account
+  belongs_to :target_account, class_name: 'Account'
+
+  before_validation :set_target_account
+  before_validation :set_followers_count
+
+  validates :acct, presence: true, domain: { acct: true }
+  validate :validate_migration_cooldown
+  validate :validate_target_account
+
+  scope :within_cooldown, ->(now = Time.now.utc) { where(arel_table[:created_at].gteq(now - COOLDOWN_PERIOD)) }
+
+  attr_accessor :current_password, :current_username
+
+  def save_with_challenge(current_user)
+    if current_user.encrypted_password.present?
+      errors.add(:current_password, :invalid) unless current_user.valid_password?(current_password)
+    else
+      errors.add(:current_username, :invalid) unless account.username == current_username
+    end
+
+    return false unless errors.empty?
+
+    save
+  end
+
+  def cooldown_at
+    created_at + COOLDOWN_PERIOD
+  end
+
+  private
+
+  def set_target_account
+    self.target_account = ResolveAccountService.new.call(acct)
+  rescue Goldfinger::Error, HTTP::Error, OpenSSL::SSL::SSLError, Mastodon::Error
+    # Validation will take care of it
+  end
+
+  def set_followers_count
+    self.followers_count = account.followers_count
+  end
+
+  def validate_target_account
+    if target_account.nil?
+      errors.add(:acct, I18n.t('migrations.errors.not_found'))
+    else
+      errors.add(:acct, I18n.t('migrations.errors.missing_also_known_as')) unless target_account.also_known_as.include?(ActivityPub::TagManager.instance.uri_for(account))
+      errors.add(:acct, I18n.t('migrations.errors.already_moved')) if account.moved_to_account_id.present? && account.moved_to_account_id == target_account.id
+      errors.add(:acct, I18n.t('migrations.errors.move_to_self')) if account.id == target_account.id
+    end
+  end
+
+  def validate_migration_cooldown
+    errors.add(:base, I18n.t('migrations.errors.on_cooldown')) if account.migrations.within_cooldown.exists?
+  end
+end
diff --git a/app/models/concerns/account_associations.rb b/app/models/concerns/account_associations.rb
index 1db7771c7..c9cc5c610 100644
--- a/app/models/concerns/account_associations.rb
+++ b/app/models/concerns/account_associations.rb
@@ -52,6 +52,8 @@ module AccountAssociations
 
     # Account migrations
     belongs_to :moved_to_account, class_name: 'Account', optional: true
+    has_many :migrations, class_name: 'AccountMigration', dependent: :destroy, inverse_of: :account
+    has_many :aliases, class_name: 'AccountAlias', dependent: :destroy, inverse_of: :account
 
     # Hashtags
     has_and_belongs_to_many :tags
diff --git a/app/models/form/migration.rb b/app/models/form/migration.rb
deleted file mode 100644
index c2a8655e1..000000000
--- a/app/models/form/migration.rb
+++ /dev/null
@@ -1,25 +0,0 @@
-# frozen_string_literal: true
-
-class Form::Migration
-  include ActiveModel::Validations
-
-  attr_accessor :acct, :account
-
-  def initialize(attrs = {})
-    @account = attrs[:account]
-    @acct    = attrs[:account].acct unless @account.nil?
-    @acct    = attrs[:acct].gsub(/\A@/, '').strip unless attrs[:acct].nil?
-  end
-
-  def valid?
-    return false unless super
-    set_account
-    errors.empty?
-  end
-
-  private
-
-  def set_account
-    self.account = (ResolveAccountService.new.call(acct) if account.nil? && acct.present?)
-  end
-end
diff --git a/app/models/remote_follow.rb b/app/models/remote_follow.rb
index 52dd3f67b..5ea535287 100644
--- a/app/models/remote_follow.rb
+++ b/app/models/remote_follow.rb
@@ -49,7 +49,7 @@ class RemoteFollow
   end
 
   def fetch_template!
-    return missing_resource if acct.blank?
+    return missing_resource_error if acct.blank?
 
     _, domain = acct.split('@')
 
diff --git a/app/models/user.rb b/app/models/user.rb
index b48455802..9a19a53b3 100644
--- a/app/models/user.rb
+++ b/app/models/user.rb
@@ -168,7 +168,7 @@ class User < ApplicationRecord
   end
 
   def functional?
-    confirmed? && approved? && !disabled? && !account.suspended?
+    confirmed? && approved? && !disabled? && !account.suspended? && account.moved_to_account_id.nil?
   end
 
   def unconfirmed_or_pending?
diff --git a/app/serializers/activitypub/move_serializer.rb b/app/serializers/activitypub/move_serializer.rb
new file mode 100644
index 000000000..5675875fa
--- /dev/null
+++ b/app/serializers/activitypub/move_serializer.rb
@@ -0,0 +1,26 @@
+# frozen_string_literal: true
+
+class ActivityPub::MoveSerializer < ActivityPub::Serializer
+  attributes :id, :type, :target, :actor
+  attribute :virtual_object, key: :object
+
+  def id
+    [ActivityPub::TagManager.instance.uri_for(object.account), '#moves/', object.id].join
+  end
+
+  def type
+    'Move'
+  end
+
+  def target
+    ActivityPub::TagManager.instance.uri_for(object.target_account)
+  end
+
+  def virtual_object
+    ActivityPub::TagManager.instance.uri_for(object.account)
+  end
+
+  def actor
+    ActivityPub::TagManager.instance.uri_for(object.account)
+  end
+end
diff --git a/app/views/auth/registrations/_status.html.haml b/app/views/auth/registrations/_status.html.haml
index b38a83d67..47112dae0 100644
--- a/app/views/auth/registrations/_status.html.haml
+++ b/app/views/auth/registrations/_status.html.haml
@@ -1,16 +1,22 @@
 %h3= t('auth.status.account_status')
 
-- if @user.account.suspended?
-  %span.negative-hint= t('user_mailer.warning.explanation.suspend')
-- elsif @user.disabled?
-  %span.negative-hint= t('user_mailer.warning.explanation.disable')
-- elsif @user.account.silenced?
-  %span.warning-hint= t('user_mailer.warning.explanation.silence')
-- elsif !@user.confirmed?
-  %span.warning-hint= t('auth.status.confirming')
-- elsif !@user.approved?
-  %span.warning-hint= t('auth.status.pending')
-- else
-  %span.positive-hint= t('auth.status.functional')
+.simple_form
+  %p.hint
+    - if @user.account.suspended?
+      %span.negative-hint= t('user_mailer.warning.explanation.suspend')
+    - elsif @user.disabled?
+      %span.negative-hint= t('user_mailer.warning.explanation.disable')
+    - elsif @user.account.silenced?
+      %span.warning-hint= t('user_mailer.warning.explanation.silence')
+    - elsif !@user.confirmed?
+      %span.warning-hint= t('auth.status.confirming')
+      = link_to t('auth.didnt_get_confirmation'), new_user_confirmation_path
+    - elsif !@user.approved?
+      %span.warning-hint= t('auth.status.pending')
+    - elsif @user.account.moved_to_account_id.present?
+      %span.positive-hint= t('auth.status.redirecting_to', acct: @user.account.moved_to_account.acct)
+      = link_to t('migrations.cancel'), settings_migration_path
+    - else
+      %span.positive-hint= t('auth.status.functional')
 
 %hr.spacer/
diff --git a/app/views/auth/registrations/edit.html.haml b/app/views/auth/registrations/edit.html.haml
index 710ee5c68..885171c58 100644
--- a/app/views/auth/registrations/edit.html.haml
+++ b/app/views/auth/registrations/edit.html.haml
@@ -13,7 +13,7 @@
       .fields-row__column.fields-group.fields-row__column-6
         = f.input :email, wrapper: :with_label, input_html: { 'aria-label' => t('simple_form.labels.defaults.email') }, required: true, disabled: current_account.suspended?
       .fields-row__column.fields-group.fields-row__column-6
-        = f.input :current_password, wrapper: :with_label, input_html: { 'aria-label' => t('simple_form.labels.defaults.current_password'), :autocomplete => 'off' }, required: true, disabled: current_account.suspended?
+        = f.input :current_password, wrapper: :with_label, input_html: { 'aria-label' => t('simple_form.labels.defaults.current_password'), :autocomplete => 'off' }, required: true, disabled: current_account.suspended?, hint: false
 
     .fields-row
       .fields-row__column.fields-group.fields-row__column-6
diff --git a/app/views/settings/aliases/index.html.haml b/app/views/settings/aliases/index.html.haml
new file mode 100644
index 000000000..5b6986368
--- /dev/null
+++ b/app/views/settings/aliases/index.html.haml
@@ -0,0 +1,29 @@
+- content_for :page_title do
+  = t('settings.aliases')
+
+= simple_form_for @alias, url: settings_aliases_path do |f|
+  = render 'shared/error_messages', object: @alias
+
+  %p.hint= t('aliases.hint_html')
+
+  %hr.spacer/
+
+  .fields-group
+    = f.input :acct, wrapper: :with_block_label, input_html: { autocapitalize: 'none', autocorrect: 'off' }
+
+  .actions
+    = f.button :button, t('aliases.add_new'), type: :submit, class: 'button'
+
+%hr.spacer/
+
+.table-wrapper
+  %table.table.inline-table
+    %thead
+      %tr
+        %th= t('simple_form.labels.account_alias.acct')
+        %th
+    %tbody
+      - @aliases.each do |account_alias|
+        %tr
+          %td= account_alias.acct
+          %td= table_link_to 'trash', t('aliases.remove'), settings_alias_path(account_alias), data: { method: :delete }
diff --git a/app/views/settings/exports/show.html.haml b/app/views/settings/exports/show.html.haml
index b13cea976..76ff76bd9 100644
--- a/app/views/settings/exports/show.html.haml
+++ b/app/views/settings/exports/show.html.haml
@@ -37,12 +37,16 @@
         %td= number_with_delimiter @export.total_domain_blocks
         %td= table_link_to 'download', t('exports.csv'), settings_exports_domain_blocks_path(format: :csv)
 
+%hr.spacer/
+
 %p.muted-hint= t('exports.archive_takeout.hint_html')
 
 - if policy(:backup).create?
   %p= link_to t('exports.archive_takeout.request'), settings_export_path, class: 'button', method: :post
 
 - unless @backups.empty?
+  %hr.spacer/
+
   .table-wrapper
     %table.table
       %thead
diff --git a/app/views/settings/migrations/show.html.haml b/app/views/settings/migrations/show.html.haml
index c69061d50..1e5c47726 100644
--- a/app/views/settings/migrations/show.html.haml
+++ b/app/views/settings/migrations/show.html.haml
@@ -1,17 +1,85 @@
 - content_for :page_title do
   = t('settings.migrate')
 
-= simple_form_for @migration, as: :migration, url: settings_migration_path, html: { method: :put } do |f|
-  - if @migration.account
-    %p.hint= t('migrations.currently_redirecting')
+.simple_form
+  - if current_account.moved_to_account.present?
+    .fields-row
+      .fields-row__column.fields-group.fields-row__column-6
+        = render 'application/card', account: current_account.moved_to_account
+      .fields-row__column.fields-group.fields-row__column-6
+        %p.hint
+          %span.positive-hint= t('migrations.redirecting_to', acct: current_account.moved_to_account.acct)
 
-    .fields-group
-      = render partial: 'application/card', locals: { account: @migration.account }
+        %p.hint= t('migrations.cancel_explanation')
+
+        %p.hint= link_to t('migrations.cancel'), cancel_settings_migration_path, data: { method: :post }
+  - else
+    %p.hint
+      %span.positive-hint= t('migrations.not_redirecting')
+
+%hr.spacer/
+
+%h3= t 'migrations.proceed_with_move'
+
+= simple_form_for @migration, url: settings_migration_path do |f|
+  - if on_cooldown?
+    %span.warning-hint= t('migrations.on_cooldown', count: ((@cooldown.cooldown_at - Time.now.utc) / 1.day.seconds).ceil)
+  - else
+    %p.hint= t('migrations.warning.before')
+
+    %ul.hint
+      %li.warning-hint= t('migrations.warning.followers')
+      %li.warning-hint= t('migrations.warning.other_data')
+      %li.warning-hint= t('migrations.warning.backreference_required')
+      %li.warning-hint= t('migrations.warning.cooldown')
+      %li.warning-hint= t('migrations.warning.disabled_account')
+
+  %hr.spacer/
 
   = render 'shared/error_messages', object: @migration
 
-  .fields-group
-    = f.input :acct, placeholder: t('migrations.acct')
+  .fields-row
+    .fields-row__column.fields-group.fields-row__column-6
+      = f.input :acct, wrapper: :with_block_label, input_html: { autocapitalize: 'none', autocorrect: 'off' }, disabled: on_cooldown?
+
+    .fields-row__column.fields-group.fields-row__column-6
+      - if current_user.encrypted_password.present?
+        = f.input :current_password, wrapper: :with_block_label, input_html: { :autocomplete => 'off' }, required: true, disabled: on_cooldown?
+      - else
+        = f.input :current_username, wrapper: :with_block_label, input_html: { :autocomplete => 'off' }, required: true, disabled: on_cooldown?
 
   .actions
-    = f.button :button, t('migrations.proceed'), type: :submit, class: 'negative'
+    = f.button :button, t('migrations.proceed_with_move'), type: :submit, class: 'button button--destructive', disabled: on_cooldown?
+
+- unless @migrations.empty?
+  %hr.spacer/
+
+  %h3= t 'migrations.past_migrations'
+
+  %hr.spacer/
+
+  .table-wrapper
+    %table.table.inline-table
+      %thead
+        %tr
+          %th= t('migrations.acct')
+          %th= t('migrations.followers_count')
+          %th
+      %tbody
+        - @migrations.each do |migration|
+          %tr
+            %td
+              - if migration.target_account.present?
+                = compact_account_link_to migration.target_account
+              - else
+                = migration.acct
+
+            %td= number_with_delimiter migration.followers_count
+
+            %td
+              %time.time-ago{ datetime: migration.created_at.iso8601, title: l(migration.created_at) }= l(migration.created_at)
+
+%hr.spacer/
+
+%h3= t 'migrations.incoming_migrations'
+%p.muted-hint= t('migrations.incoming_migrations_html', path: settings_aliases_path)
diff --git a/app/views/settings/profiles/show.html.haml b/app/views/settings/profiles/show.html.haml
index f042011d6..6929f54f3 100644
--- a/app/views/settings/profiles/show.html.haml
+++ b/app/views/settings/profiles/show.html.haml
@@ -60,6 +60,11 @@
 %h6= t('auth.migrate_account')
 %p.muted-hint= t('auth.migrate_account_html', path: settings_migration_path)
 
+%hr.spacer/
+
+%h6= t 'migrations.incoming_migrations'
+%p.muted-hint= t('migrations.incoming_migrations_html', path: settings_aliases_path)
+
 - if open_deletion?
   %hr.spacer/
 
diff --git a/app/workers/activitypub/move_distribution_worker.rb b/app/workers/activitypub/move_distribution_worker.rb
new file mode 100644
index 000000000..396d5258f
--- /dev/null
+++ b/app/workers/activitypub/move_distribution_worker.rb
@@ -0,0 +1,32 @@
+# frozen_string_literal: true
+
+class ActivityPub::MoveDistributionWorker
+  include Sidekiq::Worker
+  include Payloadable
+
+  sidekiq_options queue: 'push'
+
+  def perform(migration_id)
+    @migration = AccountMigration.find(migration_id)
+
+    ActivityPub::DeliveryWorker.push_bulk(inboxes) do |inbox_url|
+      [signed_payload, @account.id, inbox_url]
+    end
+
+    ActivityPub::DeliveryWorker.push_bulk(Relay.enabled.pluck(:inbox_url)) do |inbox_url|
+      [signed_payload, @account.id, inbox_url]
+    end
+  rescue ActiveRecord::RecordNotFound
+    true
+  end
+
+  private
+
+  def inboxes
+    @inboxes ||= @migration.account.followers.inboxes
+  end
+
+  def signed_payload
+    @signed_payload ||= Oj.dump(serialize_payload(@migration, ActivityPub::MoveSerializer, signer: @account))
+  end
+end
diff --git a/config/locales/en.yml b/config/locales/en.yml
index dabb679e7..c29c7f871 100644
--- a/config/locales/en.yml
+++ b/config/locales/en.yml
@@ -554,6 +554,12 @@ en:
     new_trending_tag:
       body: 'The hashtag #%{name} is trending today, but has not been previously reviewed. It will not be displayed publicly unless you allow it to, or just save the form as it is to never hear about it again.'
       subject: New hashtag up for review on %{instance} (#%{name})
+  aliases:
+    add_new: Create alias
+    created_msg: Successfully created a new alias. You can now initiate the move from the old account.
+    deleted_msg: Successfully remove the alias. Moving from that account to this one will no longer be possible.
+    hint_html: If you want to move from another account to this one, here you can create an alias, which is required before you can proceed with moving followers from the old account to this one. This action by itself is <strong>harmless and reversible</strong>. <strong>The account migration is initiated from the old account</strong>.
+    remove: Unlink alias
   appearance:
     advanced_web_interface: Advanced web interface
     advanced_web_interface_hint: 'If you want to make use of your entire screen width, the advanced web interface allows you to configure many different columns to see as much information at the same time as you want: Home, notifications, federated timeline, any number of lists and hashtags.'
@@ -613,6 +619,7 @@ en:
       confirming: Waiting for e-mail confirmation to be completed.
       functional: Your account is fully operational.
       pending: Your application is pending review by our staff. This may take some time. You will receive an e-mail if your application is approved.
+      redirecting_to: Your account is inactive because it is currently redirecting to %{acct}.
     trouble_logging_in: Trouble logging in?
   authorize_follow:
     already_following: You are already following this account
@@ -801,10 +808,32 @@ en:
       images_and_video: Cannot attach a video to a status that already contains images
       too_many: Cannot attach more than 4 files
   migrations:
-    acct: username@domain of the new account
-    currently_redirecting: 'Your profile is set to redirect to:'
-    proceed: Save
-    updated_msg: Your account migration setting successfully updated!
+    acct: Moved to
+    cancel: Cancel redirect
+    cancel_explanation: Cancelling the redirect will re-activate your current account, but will not bring back followers that have been moved to that account.
+    cancelled_msg: Successfully cancelled the redirect.
+    errors:
+      already_moved: is the same account you have already moved to
+      missing_also_known_as: is not back-referencing this account
+      move_to_self: cannot be current account
+      not_found: could not be found
+      on_cooldown: You are on cooldown
+    followers_count: Followers at time of move
+    incoming_migrations: Moving from a different account
+    incoming_migrations_html: To move from another account to this one, first you need to <a href="%{path}">create an account alias</a>.
+    moved_msg: Your account is now redirecting to %{acct} and your followers are being moved over.
+    not_redirecting: Your account is not redirecting to any other account currently.
+    on_cooldown: You have recently migrated your account. This function will become available again in %{count} days.
+    past_migrations: Past migrations
+    proceed_with_move: Move followers
+    redirecting_to: Your account is redirecting to %{acct}.
+    warning:
+      backreference_required: The new account must first be configured to back-reference this one
+      before: 'Before proceeding, please read these notes carefully:'
+      cooldown: After moving there is a cooldown period during which you will not be able to move again
+      disabled_account: Your current account will not be fully usable afterwards. However, you will have access to data export as well as re-activation.
+      followers: This action will move all followers from the current account to the new account
+      other_data: No other data will be moved automatically
   moderation:
     title: Moderation
   notification_mailer:
@@ -950,6 +979,7 @@ en:
   settings:
     account: Account
     account_settings: Account settings
+    aliases: Account aliases
     appearance: Appearance
     authorized_apps: Authorized apps
     back: Back to Mastodon
diff --git a/config/locales/simple_form.en.yml b/config/locales/simple_form.en.yml
index c9ffcfc13..3d909e999 100644
--- a/config/locales/simple_form.en.yml
+++ b/config/locales/simple_form.en.yml
@@ -2,6 +2,10 @@
 en:
   simple_form:
     hints:
+      account_alias:
+        acct: Specify the username@domain of the account you want to move from
+      account_migration:
+        acct: Specify the username@domain of the account you want to move to
       account_warning_preset:
         text: You can use toot syntax, such as URLs, hashtags and mentions
       admin_account_action:
@@ -15,6 +19,8 @@ en:
         avatar: PNG, GIF or JPG. At most %{size}. Will be downscaled to %{dimensions}px
         bot: This account mainly performs automated actions and might not be monitored
         context: One or multiple contexts where the filter should apply
+        current_password: For security purposes please enter the password of the current account
+        current_username: To confirm, please enter the username of the current account
         digest: Only sent after a long period of inactivity and only if you have received any personal messages in your absence
         discoverable: The profile directory is another way by which your account can reach a wider audience
         email: You will be sent a confirmation e-mail
@@ -60,6 +66,10 @@ en:
         fields:
           name: Label
           value: Content
+      account_alias:
+        acct: Handle of the old account
+      account_migration:
+        acct: Handle of the new account
       account_warning_preset:
         text: Preset text
       admin_account_action:
diff --git a/config/navigation.rb b/config/navigation.rb
index 38668bbf7..32c299143 100644
--- a/config/navigation.rb
+++ b/config/navigation.rb
@@ -5,7 +5,7 @@ SimpleNavigation::Configuration.run do |navigation|
     n.item :web, safe_join([fa_icon('chevron-left fw'), t('settings.back')]), root_url
 
     n.item :profile, safe_join([fa_icon('user fw'), t('settings.profile')]), settings_profile_url, if: -> { current_user.functional? } do |s|
-      s.item :profile, safe_join([fa_icon('pencil fw'), t('settings.appearance')]), settings_profile_url, highlights_on: %r{/settings/profile|/settings/migration}
+      s.item :profile, safe_join([fa_icon('pencil fw'), t('settings.appearance')]), settings_profile_url
       s.item :featured_tags, safe_join([fa_icon('hashtag fw'), t('settings.featured_tags')]), settings_featured_tags_url
       s.item :identity_proofs, safe_join([fa_icon('key fw'), t('settings.identity_proofs')]), settings_identity_proofs_path, highlights_on: %r{/settings/identity_proofs*}, if: proc { current_account.identity_proofs.exists? }
     end
@@ -20,13 +20,13 @@ SimpleNavigation::Configuration.run do |navigation|
     n.item :filters, safe_join([fa_icon('filter fw'), t('filters.index.title')]), filters_path, highlights_on: %r{/filters}, if: -> { current_user.functional? }
 
     n.item :security, safe_join([fa_icon('lock fw'), t('settings.account')]), edit_user_registration_url do |s|
-      s.item :password, safe_join([fa_icon('lock fw'), t('settings.account_settings')]), edit_user_registration_url, highlights_on: %r{/auth/edit|/settings/delete}
+      s.item :password, safe_join([fa_icon('lock fw'), t('settings.account_settings')]), edit_user_registration_url, highlights_on: %r{/auth/edit|/settings/delete|/settings/migration|/settings/aliases}
       s.item :two_factor_authentication, safe_join([fa_icon('mobile fw'), t('settings.two_factor_authentication')]), settings_two_factor_authentication_url, highlights_on: %r{/settings/two_factor_authentication}
       s.item :authorized_apps, safe_join([fa_icon('list fw'), t('settings.authorized_apps')]), oauth_authorized_applications_url
     end
 
-    n.item :data, safe_join([fa_icon('cloud-download fw'), t('settings.import_and_export')]), settings_export_url, if: -> { current_user.functional? } do |s|
-      s.item :import, safe_join([fa_icon('cloud-upload fw'), t('settings.import')]), settings_import_url
+    n.item :data, safe_join([fa_icon('cloud-download fw'), t('settings.import_and_export')]), settings_export_url do |s|
+      s.item :import, safe_join([fa_icon('cloud-upload fw'), t('settings.import')]), settings_import_url, if: -> { current_user.functional? }
       s.item :export, safe_join([fa_icon('cloud-download fw'), t('settings.export')]), settings_export_url
     end
 
diff --git a/config/routes.rb b/config/routes.rb
index dcfa079a0..37e0cbdee 100644
--- a/config/routes.rb
+++ b/config/routes.rb
@@ -134,8 +134,14 @@ Rails.application.routes.draw do
     end
 
     resource :delete, only: [:show, :destroy]
-    resource :migration, only: [:show, :update]
 
+    resource :migration, only: [:show, :create] do
+      collection do
+        post :cancel
+      end
+    end
+
+    resources :aliases, only: [:index, :create, :destroy]
     resources :sessions, only: [:destroy]
     resources :featured_tags, only: [:index, :create, :destroy]
   end
diff --git a/db/migrate/20190914202517_create_account_migrations.rb b/db/migrate/20190914202517_create_account_migrations.rb
new file mode 100644
index 000000000..cb9d71c09
--- /dev/null
+++ b/db/migrate/20190914202517_create_account_migrations.rb
@@ -0,0 +1,12 @@
+class CreateAccountMigrations < ActiveRecord::Migration[5.2]
+  def change
+    create_table :account_migrations do |t|
+      t.belongs_to :account, foreign_key: { on_delete: :cascade }
+      t.string :acct, null: false, default: ''
+      t.bigint :followers_count, null: false, default: 0
+      t.belongs_to :target_account, foreign_key: { to_table: :accounts, on_delete: :nullify }
+
+      t.timestamps
+    end
+  end
+end
diff --git a/db/migrate/20190915194355_create_account_aliases.rb b/db/migrate/20190915194355_create_account_aliases.rb
new file mode 100644
index 000000000..32ce031d9
--- /dev/null
+++ b/db/migrate/20190915194355_create_account_aliases.rb
@@ -0,0 +1,11 @@
+class CreateAccountAliases < ActiveRecord::Migration[5.2]
+  def change
+    create_table :account_aliases do |t|
+      t.belongs_to :account, foreign_key: { on_delete: :cascade }
+      t.string :acct, null: false, default: ''
+      t.string :uri, null: false, default: ''
+
+      t.timestamps
+    end
+  end
+end
diff --git a/db/schema.rb b/db/schema.rb
index 749f79dee..fabeb16f3 100644
--- a/db/schema.rb
+++ b/db/schema.rb
@@ -15,6 +15,15 @@ ActiveRecord::Schema.define(version: 2019_09_17_213523) do
   # These are extensions that must be enabled in order to support this database
   enable_extension "plpgsql"
 
+  create_table "account_aliases", force: :cascade do |t|
+    t.bigint "account_id"
+    t.string "acct", default: "", null: false
+    t.string "uri", default: "", null: false
+    t.datetime "created_at", null: false
+    t.datetime "updated_at", null: false
+    t.index ["account_id"], name: "index_account_aliases_on_account_id"
+  end
+
   create_table "account_conversations", force: :cascade do |t|
     t.bigint "account_id"
     t.bigint "conversation_id"
@@ -49,6 +58,17 @@ ActiveRecord::Schema.define(version: 2019_09_17_213523) do
     t.index ["account_id"], name: "index_account_identity_proofs_on_account_id"
   end
 
+  create_table "account_migrations", force: :cascade do |t|
+    t.bigint "account_id"
+    t.string "acct", default: "", null: false
+    t.bigint "followers_count", default: 0, null: false
+    t.bigint "target_account_id"
+    t.datetime "created_at", null: false
+    t.datetime "updated_at", null: false
+    t.index ["account_id"], name: "index_account_migrations_on_account_id"
+    t.index ["target_account_id"], name: "index_account_migrations_on_target_account_id"
+  end
+
   create_table "account_moderation_notes", force: :cascade do |t|
     t.text "content", null: false
     t.bigint "account_id", null: false
@@ -768,10 +788,13 @@ ActiveRecord::Schema.define(version: 2019_09_17_213523) do
     t.index ["user_id"], name: "index_web_settings_on_user_id", unique: true
   end
 
+  add_foreign_key "account_aliases", "accounts", on_delete: :cascade
   add_foreign_key "account_conversations", "accounts", on_delete: :cascade
   add_foreign_key "account_conversations", "conversations", on_delete: :cascade
   add_foreign_key "account_domain_blocks", "accounts", name: "fk_206c6029bd", on_delete: :cascade
   add_foreign_key "account_identity_proofs", "accounts", on_delete: :cascade
+  add_foreign_key "account_migrations", "accounts", column: "target_account_id", on_delete: :nullify
+  add_foreign_key "account_migrations", "accounts", on_delete: :cascade
   add_foreign_key "account_moderation_notes", "accounts"
   add_foreign_key "account_moderation_notes", "accounts", column: "target_account_id"
   add_foreign_key "account_pins", "accounts", column: "target_account_id", on_delete: :cascade
diff --git a/spec/controllers/settings/migrations_controller_spec.rb b/spec/controllers/settings/migrations_controller_spec.rb
index 4d814a45e..36e4ba86e 100644
--- a/spec/controllers/settings/migrations_controller_spec.rb
+++ b/spec/controllers/settings/migrations_controller_spec.rb
@@ -21,6 +21,7 @@ describe Settings::MigrationsController do
 
       let(:user) { Fabricate(:user, account: account) }
       let(:account) { Fabricate(:account, moved_to_account: moved_to_account) }
+
       before { sign_in user, scope: :user }
 
       context 'when user does not have moved to account' do
@@ -32,7 +33,7 @@ describe Settings::MigrationsController do
         end
       end
 
-      context 'when user does not have moved to account' do
+      context 'when user has a moved to account' do
         let(:moved_to_account) { Fabricate(:account) }
 
         it 'renders show page' do
@@ -43,21 +44,22 @@ describe Settings::MigrationsController do
     end
   end
 
-  describe 'PUT #update' do
+  describe 'POST #create' do
     context 'when user is not sign in' do
-      subject { put :update }
+      subject { post :create }
 
       it_behaves_like 'authenticate user'
     end
 
     context 'when user is sign in' do
-      subject { put :update, params: { migration: { acct: acct } } }
+      subject { post :create, params: { account_migration: { acct: acct, current_password: '12345678' } } }
+
+      let(:user) { Fabricate(:user, password: '12345678') }
 
-      let(:user) { Fabricate(:user) }
       before { sign_in user, scope: :user }
 
       context 'when migration account is changed' do
-        let(:acct) { Fabricate(:account) }
+        let(:acct) { Fabricate(:account, also_known_as: [ActivityPub::TagManager.instance.uri_for(user.account)]) }
 
         it 'updates moved to account' do
           is_expected.to redirect_to settings_migration_path
diff --git a/spec/fabricators/account_alias_fabricator.rb b/spec/fabricators/account_alias_fabricator.rb
new file mode 100644
index 000000000..94dde9bb8
--- /dev/null
+++ b/spec/fabricators/account_alias_fabricator.rb
@@ -0,0 +1,5 @@
+Fabricator(:account_alias) do
+  account
+  acct 'test@example.com'
+  uri 'https://example.com/users/test'
+end
diff --git a/spec/fabricators/account_migration_fabricator.rb b/spec/fabricators/account_migration_fabricator.rb
new file mode 100644
index 000000000..3b3fc2077
--- /dev/null
+++ b/spec/fabricators/account_migration_fabricator.rb
@@ -0,0 +1,6 @@
+Fabricator(:account_migration) do
+  account
+  target_account
+  followers_count 1234
+  acct 'test@example.com'
+end
diff --git a/spec/models/account_alias_spec.rb b/spec/models/account_alias_spec.rb
new file mode 100644
index 000000000..27ec215aa
--- /dev/null
+++ b/spec/models/account_alias_spec.rb
@@ -0,0 +1,5 @@
+require 'rails_helper'
+
+RSpec.describe AccountAlias, type: :model do
+
+end
diff --git a/spec/models/account_migration_spec.rb b/spec/models/account_migration_spec.rb
new file mode 100644
index 000000000..8461b4b28
--- /dev/null
+++ b/spec/models/account_migration_spec.rb
@@ -0,0 +1,5 @@
+require 'rails_helper'
+
+RSpec.describe AccountMigration, type: :model do
+
+end
-- 
cgit 


From 18b451c0e6cf6a927a22084f94b423982de0ee8b Mon Sep 17 00:00:00 2001
From: ThibG <thib@sitedethib.com>
Date: Fri, 27 Sep 2019 21:13:51 +0200
Subject: Change silences to always require approval on follow (#11975)
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

* Change silenced accounts to require approval on follow

* Also require approval for follows by people explicitly muted by target accounts

* Do not auto-accept silenced or muted accounts when switching from locked to unlocked

* Add `follow_requests_count` to verify_credentials

* Show “Follow requests” menu item if needed even if account is locked

* Add tests

* Correctly reflect that follow requests weren't auto-accepted when local account is silenced

* Accept follow requests from user-muted accounts to avoid leaking mutes
---
 app/controllers/api/v1/accounts_controller.rb      |  2 +-
 .../mastodon/features/getting_started/index.js     |  8 ++---
 app/lib/activitypub/activity/follow.rb             |  2 +-
 .../rest/credential_account_serializer.rb          |  1 +
 app/services/follow_service.rb                     |  2 +-
 app/services/update_account_service.rb             |  4 ++-
 spec/lib/activitypub/activity/follow_spec.rb       | 30 +++++++++++++++++
 spec/services/follow_service_spec.rb               | 27 +++++++++++++++
 spec/services/update_account_service_spec.rb       | 38 ++++++++++++++++++++++
 9 files changed, 105 insertions(+), 9 deletions(-)
 create mode 100644 spec/services/update_account_service_spec.rb

(limited to 'spec')

diff --git a/app/controllers/api/v1/accounts_controller.rb b/app/controllers/api/v1/accounts_controller.rb
index b306e8e8c..c12e1c12e 100644
--- a/app/controllers/api/v1/accounts_controller.rb
+++ b/app/controllers/api/v1/accounts_controller.rb
@@ -33,7 +33,7 @@ class Api::V1::AccountsController < Api::BaseController
   def follow
     FollowService.new.call(current_user.account, @account, reblogs: truthy_param?(:reblogs))
 
-    options = @account.locked? ? {} : { following_map: { @account.id => { reblogs: truthy_param?(:reblogs) } }, requested_map: { @account.id => false } }
+    options = @account.locked? || current_user.account.silenced? ? {} : { following_map: { @account.id => { reblogs: truthy_param?(:reblogs) } }, requested_map: { @account.id => false } }
 
     render json: @account, serializer: REST::RelationshipSerializer, relationships: relationships(options)
   end
diff --git a/app/javascript/mastodon/features/getting_started/index.js b/app/javascript/mastodon/features/getting_started/index.js
index f6d90580b..67ec7665b 100644
--- a/app/javascript/mastodon/features/getting_started/index.js
+++ b/app/javascript/mastodon/features/getting_started/index.js
@@ -77,16 +77,14 @@ class GettingStarted extends ImmutablePureComponent {
   };
 
   componentDidMount () {
-    const { myAccount, fetchFollowRequests, multiColumn } = this.props;
+    const { fetchFollowRequests, multiColumn } = this.props;
 
     if (!multiColumn && window.innerWidth >= NAVIGATION_PANEL_BREAKPOINT) {
       this.context.router.history.replace('/timelines/home');
       return;
     }
 
-    if (myAccount.get('locked')) {
-      fetchFollowRequests();
-    }
+    fetchFollowRequests();
   }
 
   render () {
@@ -134,7 +132,7 @@ class GettingStarted extends ImmutablePureComponent {
 
     height += 48*3;
 
-    if (myAccount.get('locked')) {
+    if (myAccount.get('locked') || unreadFollowRequests > 0) {
       navItems.push(<ColumnLink key={i++} icon='user-plus' text={intl.formatMessage(messages.follow_requests)} badge={badgeDisplay(unreadFollowRequests, 40)} to='/follow_requests' />);
       height += 48;
     }
diff --git a/app/lib/activitypub/activity/follow.rb b/app/lib/activitypub/activity/follow.rb
index 28f1da19f..ec92f4255 100644
--- a/app/lib/activitypub/activity/follow.rb
+++ b/app/lib/activitypub/activity/follow.rb
@@ -21,7 +21,7 @@ class ActivityPub::Activity::Follow < ActivityPub::Activity
 
     follow_request = FollowRequest.create!(account: @account, target_account: target_account, uri: @json['id'])
 
-    if target_account.locked?
+    if target_account.locked? || @account.silenced?
       NotifyService.new.call(target_account, follow_request)
     else
       AuthorizeFollowService.new.call(@account, target_account)
diff --git a/app/serializers/rest/credential_account_serializer.rb b/app/serializers/rest/credential_account_serializer.rb
index fb195eb07..be0d763dc 100644
--- a/app/serializers/rest/credential_account_serializer.rb
+++ b/app/serializers/rest/credential_account_serializer.rb
@@ -12,6 +12,7 @@ class REST::CredentialAccountSerializer < REST::AccountSerializer
       language: user.setting_default_language,
       note: object.note,
       fields: object.fields.map(&:to_h),
+      follow_requests_count: FollowRequest.where(target_account: object).limit(40).count,
     }
   end
 end
diff --git a/app/services/follow_service.rb b/app/services/follow_service.rb
index 101acdaf9..1941c2e2d 100644
--- a/app/services/follow_service.rb
+++ b/app/services/follow_service.rb
@@ -30,7 +30,7 @@ class FollowService < BaseService
 
     ActivityTracker.increment('activity:interactions')
 
-    if target_account.locked? || target_account.activitypub?
+    if target_account.locked? || source_account.silenced? || target_account.activitypub?
       request_follow(source_account, target_account, reblogs: reblogs)
     elsif target_account.local?
       direct_follow(source_account, target_account, reblogs: reblogs)
diff --git a/app/services/update_account_service.rb b/app/services/update_account_service.rb
index 01756a73d..ebf24be37 100644
--- a/app/services/update_account_service.rb
+++ b/app/services/update_account_service.rb
@@ -20,7 +20,9 @@ class UpdateAccountService < BaseService
   private
 
   def authorize_all_follow_requests(account)
-    AuthorizeFollowWorker.push_bulk(FollowRequest.where(target_account: account).select(:account_id, :target_account_id)) do |req|
+    follow_requests = FollowRequest.where(target_account: account)
+    follow_requests = follow_requests.select { |req| !req.account.silenced? }
+    AuthorizeFollowWorker.push_bulk(follow_requests) do |req|
       [req.account_id, req.target_account_id]
     end
   end
diff --git a/spec/lib/activitypub/activity/follow_spec.rb b/spec/lib/activitypub/activity/follow_spec.rb
index 6bbacdbe6..05112cc18 100644
--- a/spec/lib/activitypub/activity/follow_spec.rb
+++ b/spec/lib/activitypub/activity/follow_spec.rb
@@ -31,6 +31,36 @@ RSpec.describe ActivityPub::Activity::Follow do
       end
     end
 
+    context 'silenced account following an unlocked account' do
+      before do
+        sender.touch(:silenced_at)
+        subject.perform
+      end
+
+      it 'does not create a follow from sender to recipient' do
+        expect(sender.following?(recipient)).to be false
+      end
+
+      it 'creates a follow request' do
+        expect(sender.requested?(recipient)).to be true
+      end
+    end
+
+    context 'unlocked account muting the sender' do
+      before do
+        recipient.mute!(sender)
+        subject.perform
+      end
+
+      it 'creates a follow from sender to recipient' do
+        expect(sender.following?(recipient)).to be true
+      end
+
+      it 'does not create a follow request' do
+        expect(sender.requested?(recipient)).to be false
+      end
+    end
+
     context 'locked account' do
       before do
         recipient.update(locked: true)
diff --git a/spec/services/follow_service_spec.rb b/spec/services/follow_service_spec.rb
index 86c85293e..ae863a9f0 100644
--- a/spec/services/follow_service_spec.rb
+++ b/spec/services/follow_service_spec.rb
@@ -30,6 +30,33 @@ RSpec.describe FollowService, type: :service do
       end
     end
 
+    describe 'unlocked account, from silenced account' do
+      let(:bob) { Fabricate(:user, email: 'bob@example.com', account: Fabricate(:account, username: 'bob')).account }
+
+      before do
+        sender.touch(:silenced_at)
+        subject.call(sender, bob.acct)
+      end
+
+      it 'creates a follow request with reblogs' do
+        expect(FollowRequest.find_by(account: sender, target_account: bob, show_reblogs: true)).to_not be_nil
+      end
+    end
+
+    describe 'unlocked account, from a muted account' do
+      let(:bob) { Fabricate(:user, email: 'bob@example.com', account: Fabricate(:account, username: 'bob')).account }
+
+      before do
+        bob.mute!(sender)
+        subject.call(sender, bob.acct)
+      end
+
+      it 'creates a following relation with reblogs' do
+        expect(sender.following?(bob)).to be true
+        expect(sender.muting_reblogs?(bob)).to be false
+      end
+    end
+
     describe 'unlocked account' do
       let(:bob) { Fabricate(:user, email: 'bob@example.com', account: Fabricate(:account, username: 'bob')).account }
 
diff --git a/spec/services/update_account_service_spec.rb b/spec/services/update_account_service_spec.rb
new file mode 100644
index 000000000..960b26891
--- /dev/null
+++ b/spec/services/update_account_service_spec.rb
@@ -0,0 +1,38 @@
+require 'rails_helper'
+
+RSpec.describe UpdateAccountService, type: :service do
+  subject { UpdateAccountService.new }
+
+  describe 'switching form locked to unlocked accounts' do
+    let(:account) { Fabricate(:account, locked: true) }
+    let(:alice)   { Fabricate(:user, email: 'alice@example.com', account: Fabricate(:account, username: 'alice')).account }
+    let(:bob)     { Fabricate(:user, email: 'bob@example.com', account: Fabricate(:account, username: 'bob')).account }
+    let(:eve)     { Fabricate(:user, email: 'eve@example.com', account: Fabricate(:account, username: 'eve')).account }
+
+    before do
+      bob.touch(:silenced_at)
+      account.mute!(eve)
+
+      FollowService.new.call(alice, account)
+      FollowService.new.call(bob, account)
+      FollowService.new.call(eve, account)
+
+      subject.call(account, { locked: false })
+    end
+
+    it 'auto-accepts pending follow requests' do
+      expect(alice.following?(account)).to be true
+      expect(alice.requested?(account)).to be false
+    end
+
+    it 'does not auto-accept pending follow requests from silenced users' do
+      expect(bob.following?(account)).to be false
+      expect(bob.requested?(account)).to be true
+    end
+
+    it 'auto-accepts pending follow requests from muted users so as to not leak mute' do
+      expect(eve.following?(account)).to be true
+      expect(eve.requested?(account)).to be false
+    end
+  end
+end
-- 
cgit 


From ab33c4df942ec3fdc4d891f3db7ac8cdd3436945 Mon Sep 17 00:00:00 2001
From: Eugen Rochko <eugen@zeonfederated.com>
Date: Sat, 28 Sep 2019 01:02:21 +0200
Subject: Add `exclude_unreviewed` param to `GET /api/v2/search` REST API
 (#11977)

Make it so normal search returns even unreviewed matches, but
autosuggestions do not.

Fix #11960
---
 app/controllers/api/v2/search_controller.rb |  2 +-
 app/javascript/mastodon/actions/compose.js  |  1 +
 app/models/tag.rb                           | 13 ++++++-------
 app/services/search_service.rb              |  3 ++-
 app/services/tag_search_service.rb          | 16 ++++++++++------
 spec/services/search_service_spec.rb        |  4 ++--
 6 files changed, 22 insertions(+), 17 deletions(-)

(limited to 'spec')

diff --git a/app/controllers/api/v2/search_controller.rb b/app/controllers/api/v2/search_controller.rb
index c14cd22d7..cbd9b551d 100644
--- a/app/controllers/api/v2/search_controller.rb
+++ b/app/controllers/api/v2/search_controller.rb
@@ -22,7 +22,7 @@ class Api::V2::SearchController < Api::BaseController
       params[:q],
       current_account,
       limit_param(RESULTS_LIMIT),
-      search_params.merge(resolve: truthy_param?(:resolve))
+      search_params.merge(resolve: truthy_param?(:resolve), exclude_unreviewed: truthy_param?(:exclude_unreviewed))
     )
   end
 
diff --git a/app/javascript/mastodon/actions/compose.js b/app/javascript/mastodon/actions/compose.js
index 79d64dc3e..8e7906c73 100644
--- a/app/javascript/mastodon/actions/compose.js
+++ b/app/javascript/mastodon/actions/compose.js
@@ -369,6 +369,7 @@ const fetchComposeSuggestionsTags = throttle((dispatch, getState, token) => {
       q: token.slice(1),
       resolve: false,
       limit: 4,
+      exclude_unreviewed: true,
     },
   }).then(({ data }) => {
     dispatch(readyComposeSuggestionsTags(token, data.hashtags));
diff --git a/app/models/tag.rb b/app/models/tag.rb
index b52b9bc9f..9aca3983f 100644
--- a/app/models/tag.rb
+++ b/app/models/tag.rb
@@ -124,16 +124,15 @@ class Tag < ApplicationRecord
       end
     end
 
-    def search_for(term, limit = 5, offset = 0)
+    def search_for(term, limit = 5, offset = 0, options = {})
       normalized_term = normalize(term.strip).mb_chars.downcase.to_s
       pattern         = sanitize_sql_like(normalized_term) + '%'
+      query           = Tag.listable.where(arel_table[:name].lower.matches(pattern))
+      query           = query.where(arel_table[:name].lower.eq(normalized_term).or(arel_table[:reviewed_at].not_eq(nil))) if options[:exclude_unreviewed]
 
-      Tag.listable
-         .where(arel_table[:name].lower.matches(pattern))
-         .where(arel_table[:name].lower.eq(normalized_term).or(arel_table[:reviewed_at].not_eq(nil)))
-         .order(Arel.sql('length(name) ASC, name ASC'))
-         .limit(limit)
-         .offset(offset)
+      query.order(Arel.sql('length(name) ASC, name ASC'))
+           .limit(limit)
+           .offset(offset)
     end
 
     def find_normalized(name)
diff --git a/app/services/search_service.rb b/app/services/search_service.rb
index a5ba5dd11..3a498dcf4 100644
--- a/app/services/search_service.rb
+++ b/app/services/search_service.rb
@@ -60,7 +60,8 @@ class SearchService < BaseService
     TagSearchService.new.call(
       @query,
       limit: @limit,
-      offset: @offset
+      offset: @offset,
+      exclude_unreviewed: @options[:exclude_unreviewed]
     )
   end
 
diff --git a/app/services/tag_search_service.rb b/app/services/tag_search_service.rb
index 47b0e876e..b78d65625 100644
--- a/app/services/tag_search_service.rb
+++ b/app/services/tag_search_service.rb
@@ -2,11 +2,12 @@
 
 class TagSearchService < BaseService
   def call(query, options = {})
-    @query  = query.strip.gsub(/\A#/, '')
-    @offset = options[:offset].to_i
-    @limit  = options[:limit].to_i
+    @query   = query.strip.gsub(/\A#/, '')
+    @offset  = options.delete(:offset).to_i
+    @limit   = options.delete(:limit).to_i
+    @options = options
 
-    results = from_elasticsearch if Chewy.enabled?
+    results   = from_elasticsearch if Chewy.enabled?
     results ||= from_database
 
     results
@@ -72,12 +73,15 @@ class TagSearchService < BaseService
       },
     }
 
-    TagsIndex.query(query).filter(filter).limit(@limit).offset(@offset).objects.compact
+    definition = TagsIndex.query(query)
+    definition = definition.filter(filter) if @options[:exclude_unreviewed]
+
+    definition.limit(@limit).offset(@offset).objects.compact
   rescue Faraday::ConnectionFailed, Parslet::ParseFailed
     nil
   end
 
   def from_database
-    Tag.search_for(@query, @limit, @offset)
+    Tag.search_for(@query, @limit, @offset, @options)
   end
 end
diff --git a/spec/services/search_service_spec.rb b/spec/services/search_service_spec.rb
index ade306ed2..739bb9cf5 100644
--- a/spec/services/search_service_spec.rb
+++ b/spec/services/search_service_spec.rb
@@ -77,10 +77,10 @@ describe SearchService, type: :service do
         it 'includes the tag in the results' do
           query = '#tag'
           tag = Tag.new
-          allow(Tag).to receive(:search_for).with('tag', 10, 0).and_return([tag])
+          allow(Tag).to receive(:search_for).with('tag', 10, 0, exclude_unreviewed: nil).and_return([tag])
 
           results = subject.call(query, nil, 10)
-          expect(Tag).to have_received(:search_for).with('tag', 10, 0)
+          expect(Tag).to have_received(:search_for).with('tag', 10, 0, exclude_unreviewed: nil)
           expect(results).to eq empty_results.merge(hashtags: [tag])
         end
         it 'does not include tag when starts with @ character' do
-- 
cgit 


From 5f69eb89e215fe7dc02cd0dc3f39b13f1945e88b Mon Sep 17 00:00:00 2001
From: Eugen Rochko <eugen@zeonfederated.com>
Date: Sun, 29 Sep 2019 21:31:51 +0200
Subject: Add a nodeinfo endpoint (#12002)

* Add nodeinfo endpoint

* dont commit stuff from my local dev

* consistant naming since we implimented 2.1 schema

* Add some additional node info stuff

* Add nodeinfo endpoint

* dont commit stuff from my local dev

* consistant naming since we implimented 2.1 schema

* expanding this to include federation info

* codeclimate feedback

* CC feedback

* using activeserializers seems like a good idea...

* get rid of draft 2.1 version

* Reimplement 2.1, also fix metaData -> metadata

* Fix metaData -> metadata here too

* Fix nodeinfo 2.1 tests

* Implement cache for monthly user aggregate

* Useless

* Remove ostatus from the list of supported protocols

* Fix nodeinfo's open_registration reading obsolete setting variable

* Only serialize domain blocks with user-facing limitations

* Do not needlessly list noop severity in nodeinfo

* Only serialize domain blocks info in nodeinfo when they are set to be displayed to everyone

* Enable caching for nodeinfo endpoints

* Fix rendering nodeinfo

* CodeClimate fixes

* Please CodeClimate

* Change InstancePresenter#active_user_count_months for clarity

* Refactor NodeInfoSerializer#metadata

* Remove nodeinfo 2.1 support as the schema doesn't exist

* Clean-up
---
 app/controllers/well_known/nodeinfo_controller.rb  | 19 ++++++++++
 app/lib/activity_tracker.rb                        |  2 +-
 app/lib/nodeinfo/adapter.rb                        |  7 ++++
 app/presenters/instance_presenter.rb               |  4 +--
 app/serializers/nodeinfo/discovery_serializer.rb   | 11 ++++++
 app/serializers/nodeinfo/serializer.rb             | 41 ++++++++++++++++++++++
 config/initializers/inflections.rb                 |  1 +
 config/routes.rb                                   |  3 ++
 .../well_known/nodeinfo_controller_spec.rb         | 36 +++++++++++++++++++
 9 files changed, 121 insertions(+), 3 deletions(-)
 create mode 100644 app/controllers/well_known/nodeinfo_controller.rb
 create mode 100644 app/lib/nodeinfo/adapter.rb
 create mode 100644 app/serializers/nodeinfo/discovery_serializer.rb
 create mode 100644 app/serializers/nodeinfo/serializer.rb
 create mode 100644 spec/controllers/well_known/nodeinfo_controller_spec.rb

(limited to 'spec')

diff --git a/app/controllers/well_known/nodeinfo_controller.rb b/app/controllers/well_known/nodeinfo_controller.rb
new file mode 100644
index 000000000..11a699ebc
--- /dev/null
+++ b/app/controllers/well_known/nodeinfo_controller.rb
@@ -0,0 +1,19 @@
+# frozen_string_literal: true
+
+module WellKnown
+  class NodeInfoController < ActionController::Base
+    include CacheConcern
+
+    before_action { response.headers['Vary'] = 'Accept' }
+
+    def index
+      expires_in 3.days, public: true
+      render_with_cache json: {}, serializer: NodeInfo::DiscoverySerializer, adapter: NodeInfo::Adapter, expires_in: 3.days, root: 'nodeinfo'
+    end
+
+    def show
+      expires_in 30.minutes, public: true
+      render_with_cache json: {}, serializer: NodeInfo::Serializer, adapter: NodeInfo::Adapter, expires_in: 30.minutes, root: 'nodeinfo'
+    end
+  end
+end
diff --git a/app/lib/activity_tracker.rb b/app/lib/activity_tracker.rb
index ae3c11b6a..81303b715 100644
--- a/app/lib/activity_tracker.rb
+++ b/app/lib/activity_tracker.rb
@@ -1,7 +1,7 @@
 # frozen_string_literal: true
 
 class ActivityTracker
-  EXPIRE_AFTER = 90.days.seconds
+  EXPIRE_AFTER = 6.months.seconds
 
   class << self
     include Redisable
diff --git a/app/lib/nodeinfo/adapter.rb b/app/lib/nodeinfo/adapter.rb
new file mode 100644
index 000000000..1b48dcb98
--- /dev/null
+++ b/app/lib/nodeinfo/adapter.rb
@@ -0,0 +1,7 @@
+# frozen_string_literal: true
+
+class NodeInfo::Adapter < ActiveModelSerializers::Adapter::Attributes
+  def self.default_key_transform
+    :camel_lower
+  end
+end
diff --git a/app/presenters/instance_presenter.rb b/app/presenters/instance_presenter.rb
index becc92c2d..c4caeaa8c 100644
--- a/app/presenters/instance_presenter.rb
+++ b/app/presenters/instance_presenter.rb
@@ -20,8 +20,8 @@ class InstancePresenter
     Rails.cache.fetch('user_count') { User.confirmed.joins(:account).merge(Account.without_suspended).count }
   end
 
-  def active_user_count
-    Rails.cache.fetch('active_user_count') { Redis.current.pfcount(*(0..3).map { |i| "activity:logins:#{i.weeks.ago.utc.to_date.cweek}" }) }
+  def active_user_count(weeks = 4)
+    Rails.cache.fetch('active_user_count') { Redis.current.pfcount(*(0...weeks).map { |i| "activity:logins:#{i.weeks.ago.utc.to_date.cweek}" }) }
   end
 
   def status_count
diff --git a/app/serializers/nodeinfo/discovery_serializer.rb b/app/serializers/nodeinfo/discovery_serializer.rb
new file mode 100644
index 000000000..07ab2a6ee
--- /dev/null
+++ b/app/serializers/nodeinfo/discovery_serializer.rb
@@ -0,0 +1,11 @@
+# frozen_string_literal: true
+
+class NodeInfo::DiscoverySerializer < ActiveModel::Serializer
+  include RoutingHelper
+
+  attribute :links
+
+  def links
+    [{ rel: 'http://nodeinfo.diaspora.software/ns/schema/2.0', href: nodeinfo_schema_url }]
+  end
+end
diff --git a/app/serializers/nodeinfo/serializer.rb b/app/serializers/nodeinfo/serializer.rb
new file mode 100644
index 000000000..1a7d7a911
--- /dev/null
+++ b/app/serializers/nodeinfo/serializer.rb
@@ -0,0 +1,41 @@
+# frozen_string_literal: true
+
+class NodeInfo::Serializer < ActiveModel::Serializer
+  include RoutingHelper
+
+  attributes :version, :software, :protocols, :usage
+
+  def version
+    '2.0'
+  end
+
+  def software
+    { name: 'mastodon', version: Mastodon::Version.to_s }
+  end
+
+  def services
+    { outbound: [], inbound: [] }
+  end
+
+  def protocols
+    %w(activitypub)
+  end
+
+  def usage
+    {
+      users: {
+        total: instance_presenter.user_count,
+        active_month: instance_presenter.active_user_count(4),
+        active_halfyear: instance_presenter.active_user_count(24),
+      },
+
+      local_posts: instance_presenter.status_count,
+    }
+  end
+
+  private
+
+  def instance_presenter
+    @instance_presenter ||= InstancePresenter.new
+  end
+end
diff --git a/config/initializers/inflections.rb b/config/initializers/inflections.rb
index bf0cb52a3..c65153b0a 100644
--- a/config/initializers/inflections.rb
+++ b/config/initializers/inflections.rb
@@ -18,4 +18,5 @@ ActiveSupport::Inflector.inflections(:en) do |inflect|
   inflect.acronym 'PubSubHubbub'
   inflect.acronym 'ActivityStreams'
   inflect.acronym 'JsonLd'
+  inflect.acronym 'NodeInfo'
 end
diff --git a/config/routes.rb b/config/routes.rb
index f1a69cf5c..e43e201a5 100644
--- a/config/routes.rb
+++ b/config/routes.rb
@@ -24,10 +24,13 @@ Rails.application.routes.draw do
   end
 
   get '.well-known/host-meta', to: 'well_known/host_meta#show', as: :host_meta, defaults: { format: 'xml' }
+  get '.well-known/nodeinfo', to: 'well_known/nodeinfo#index', as: :nodeinfo, defaults: { format: 'json' }
   get '.well-known/webfinger', to: 'well_known/webfinger#show', as: :webfinger
   get '.well-known/change-password', to: redirect('/auth/edit')
   get '.well-known/keybase-proof-config', to: 'well_known/keybase_proof_config#show'
 
+  get '/nodeinfo/2.0', to: 'well_known/nodeinfo#show', as: :nodeinfo_schema
+
   get 'manifest', to: 'manifests#show', defaults: { format: 'json' }
   get 'intent', to: 'intents#show'
   get 'custom.css', to: 'custom_css#show', as: :custom_css
diff --git a/spec/controllers/well_known/nodeinfo_controller_spec.rb b/spec/controllers/well_known/nodeinfo_controller_spec.rb
new file mode 100644
index 000000000..12e1fa415
--- /dev/null
+++ b/spec/controllers/well_known/nodeinfo_controller_spec.rb
@@ -0,0 +1,36 @@
+require 'rails_helper'
+
+describe WellKnown::NodeInfoController, type: :controller do
+  render_views
+
+  describe 'GET #index' do
+    it 'returns json document pointing to node info' do
+      get :index
+
+      expect(response).to have_http_status(200)
+      expect(response.content_type).to eq 'application/json'
+
+      json = body_as_json
+
+      expect(json[:links]).to be_an Array
+      expect(json[:links][0][:rel]).to eq 'http://nodeinfo.diaspora.software/ns/schema/2.0'
+      expect(json[:links][0][:href]).to include 'nodeinfo/2.0'
+    end
+  end
+
+  describe 'GET #show' do
+    it 'returns json document with node info properties' do
+      get :show
+
+      expect(response).to have_http_status(200)
+      expect(response.content_type).to eq 'application/json'
+
+      json = body_as_json
+
+      expect(json[:version]).to eq '2.0'
+      expect(json[:usage]).to be_a Hash
+      expect(json[:software]).to be_a Hash
+      expect(json[:protocols]).to be_an Array
+    end
+  end
+end
-- 
cgit 


From 5c42f47617d311219d06e082e4daa41e671903c8 Mon Sep 17 00:00:00 2001
From: Eugen Rochko <eugen@zeonfederated.com>
Date: Tue, 1 Oct 2019 01:19:11 +0200
Subject: Fix records not being indexed sometimes (#12024)

It's possible that after commit callbacks were not firing when
exceptions occurred in the process. Also, the default Sidekiq
strategy does not push indexing jobs immediately, which is not
necessary and could be part of the issue too.
---
 app/models/account.rb                |  2 +-
 app/models/account_stat.rb           |  2 +-
 app/models/application_record.rb     |  6 ++++++
 app/models/favourite.rb              |  2 +-
 app/models/status.rb                 |  2 +-
 app/models/tag.rb                    |  2 +-
 config/application.rb                |  1 +
 config/initializers/chewy.rb         |  5 +++--
 lib/chewy/strategy/custom_sidekiq.rb | 30 ++++++++++++++++++++++++++++++
 spec/rails_helper.rb                 |  2 +-
 10 files changed, 46 insertions(+), 8 deletions(-)
 create mode 100644 lib/chewy/strategy/custom_sidekiq.rb

(limited to 'spec')

diff --git a/app/models/account.rb b/app/models/account.rb
index 55fe53fae..01d45e36c 100644
--- a/app/models/account.rb
+++ b/app/models/account.rb
@@ -129,7 +129,7 @@ class Account < ApplicationRecord
 
   delegate :chosen_languages, to: :user, prefix: false, allow_nil: true
 
-  update_index('accounts#account', :self) if Chewy.enabled?
+  update_index('accounts#account', :self)
 
   def local?
     domain.nil?
diff --git a/app/models/account_stat.rb b/app/models/account_stat.rb
index 6d1097cec..1351f7d8a 100644
--- a/app/models/account_stat.rb
+++ b/app/models/account_stat.rb
@@ -16,7 +16,7 @@
 class AccountStat < ApplicationRecord
   belongs_to :account, inverse_of: :account_stat
 
-  update_index('accounts#account', :account) if Chewy.enabled?
+  update_index('accounts#account', :account)
 
   def increment_count!(key)
     update(attributes_for_increment(key))
diff --git a/app/models/application_record.rb b/app/models/application_record.rb
index c1b873da6..5d7d3a096 100644
--- a/app/models/application_record.rb
+++ b/app/models/application_record.rb
@@ -5,6 +5,12 @@ class ApplicationRecord < ActiveRecord::Base
 
   include Remotable
 
+  class << self
+    def update_index(_type_name, *_args, &_block)
+      super if Chewy.enabled?
+    end
+  end
+
   def boolean_with_default(key, default_value)
     value = attributes[key]
 
diff --git a/app/models/favourite.rb b/app/models/favourite.rb
index 17f8c9fa6..bf0ec4449 100644
--- a/app/models/favourite.rb
+++ b/app/models/favourite.rb
@@ -13,7 +13,7 @@
 class Favourite < ApplicationRecord
   include Paginable
 
-  update_index('statuses#status', :status) if Chewy.enabled?
+  update_index('statuses#status', :status)
 
   belongs_to :account, inverse_of: :favourites
   belongs_to :status,  inverse_of: :favourites
diff --git a/app/models/status.rb b/app/models/status.rb
index 5e7474577..078a64566 100644
--- a/app/models/status.rb
+++ b/app/models/status.rb
@@ -39,7 +39,7 @@ class Status < ApplicationRecord
   # will be based on current time instead of `created_at`
   attr_accessor :override_timestamps
 
-  update_index('statuses#status', :proper) if Chewy.enabled?
+  update_index('statuses#status', :proper)
 
   enum visibility: [:public, :unlisted, :private, :direct, :limited], _suffix: :visibility
 
diff --git a/app/models/tag.rb b/app/models/tag.rb
index 9aca3983f..82786daa8 100644
--- a/app/models/tag.rb
+++ b/app/models/tag.rb
@@ -49,7 +49,7 @@ class Tag < ApplicationRecord
 
   after_save :save_account_tag_stat
 
-  update_index('tags#tag', :self) if Chewy.enabled?
+  update_index('tags#tag', :self)
 
   def account_tag_stat
     super || build_account_tag_stat
diff --git a/config/application.rb b/config/application.rb
index 3ced81b8f..60f73f8bb 100644
--- a/config/application.rb
+++ b/config/application.rb
@@ -15,6 +15,7 @@ require_relative '../lib/mastodon/snowflake'
 require_relative '../lib/mastodon/version'
 require_relative '../lib/devise/two_factor_ldap_authenticatable'
 require_relative '../lib/devise/two_factor_pam_authenticatable'
+require_relative '../lib/chewy/strategy/custom_sidekiq'
 
 Dotenv::Railtie.load
 
diff --git a/config/initializers/chewy.rb b/config/initializers/chewy.rb
index d5347f2bf..9ff0dccc1 100644
--- a/config/initializers/chewy.rb
+++ b/config/initializers/chewy.rb
@@ -12,8 +12,9 @@ Chewy.settings = {
   sidekiq: { queue: 'pull' },
 }
 
-Chewy.root_strategy    = enabled ? :sidekiq : :bypass
-Chewy.request_strategy = enabled ? :sidekiq : :bypass
+Chewy.root_strategy              = :custom_sidekiq
+Chewy.request_strategy           = :custom_sidekiq
+Chewy.use_after_commit_callbacks = false
 
 module Chewy
   class << self
diff --git a/lib/chewy/strategy/custom_sidekiq.rb b/lib/chewy/strategy/custom_sidekiq.rb
new file mode 100644
index 000000000..3e54326ba
--- /dev/null
+++ b/lib/chewy/strategy/custom_sidekiq.rb
@@ -0,0 +1,30 @@
+# frozen_string_literal: true
+
+module Chewy
+  class Strategy
+    class CustomSidekiq < Base
+      class Worker
+        include ::Sidekiq::Worker
+
+        sidekiq_options queue: 'pull'
+
+        def perform(type, ids, options = {})
+          options[:refresh] = !Chewy.disable_refresh_async if Chewy.disable_refresh_async
+          type.constantize.import!(ids, options)
+        end
+      end
+
+      def update(type, objects, _options = {})
+        return unless Chewy.enabled?
+
+        ids = type.root.id ? Array.wrap(objects) : type.adapter.identify(objects)
+
+        return if ids.empty?
+
+        Worker.perform_async(type.name, ids)
+      end
+
+      def leave; end
+    end
+  end
+end
diff --git a/spec/rails_helper.rb b/spec/rails_helper.rb
index 3a5e7491e..6fbceca53 100644
--- a/spec/rails_helper.rb
+++ b/spec/rails_helper.rb
@@ -12,7 +12,7 @@ require 'capybara/rspec'
 Dir[Rails.root.join('spec/support/**/*.rb')].each { |f| require f }
 
 ActiveRecord::Migration.maintain_test_schema!
-WebMock.disable_net_connect!
+WebMock.disable_net_connect!(allow: Chewy.settings[:host])
 Redis.current = Redis::Namespace.new("mastodon_test#{ENV['TEST_ENV_NUMBER']}", redis: Redis.current)
 Sidekiq::Testing.inline!
 Sidekiq::Logging.logger = nil
-- 
cgit 


From 9ba40a6bfdb65f6e48eb7de07b2d55314c54fa83 Mon Sep 17 00:00:00 2001
From: Eugen Rochko <eugen@zeonfederated.com>
Date: Tue, 1 Oct 2019 04:54:10 +0200
Subject: Remove HEAD request from fetching link previews (#12028)

It is not really necessary and we need to reduce requests
---
 app/services/fetch_link_card_service.rb       |  6 ------
 spec/services/fetch_link_card_service_spec.rb | 13 +++----------
 2 files changed, 3 insertions(+), 16 deletions(-)

(limited to 'spec')

diff --git a/app/services/fetch_link_card_service.rb b/app/services/fetch_link_card_service.rb
index ac5503d46..f0b1169db 100644
--- a/app/services/fetch_link_card_service.rb
+++ b/app/services/fetch_link_card_service.rb
@@ -39,12 +39,6 @@ class FetchLinkCardService < BaseService
   def process_url
     @card ||= PreviewCard.new(url: @url)
 
-    failed = Request.new(:head, @url).perform do |res|
-      res.code != 405 && res.code != 501 && (res.code != 200 || res.mime_type != 'text/html')
-    end
-
-    return if failed
-
     Request.new(:get, @url).perform do |res|
       if res.code == 200 && res.mime_type == 'text/html'
         @html = res.body_with_limit
diff --git a/spec/services/fetch_link_card_service_spec.rb b/spec/services/fetch_link_card_service_spec.rb
index 50c60aafd..9761c5f06 100644
--- a/spec/services/fetch_link_card_service_spec.rb
+++ b/spec/services/fetch_link_card_service_spec.rb
@@ -4,20 +4,13 @@ RSpec.describe FetchLinkCardService, type: :service do
   subject { FetchLinkCardService.new }
 
   before do
-    stub_request(:head, 'http://example.xn--fiqs8s/').to_return(status: 200, headers: { 'Content-Type' => 'text/html' })
     stub_request(:get, 'http://example.xn--fiqs8s/').to_return(request_fixture('idn.txt'))
-    stub_request(:head, 'http://example.com/sjis').to_return(status: 200, headers: { 'Content-Type' => 'text/html' })
     stub_request(:get, 'http://example.com/sjis').to_return(request_fixture('sjis.txt'))
-    stub_request(:head, 'http://example.com/sjis_with_wrong_charset').to_return(status: 200, headers: { 'Content-Type' => 'text/html' })
     stub_request(:get, 'http://example.com/sjis_with_wrong_charset').to_return(request_fixture('sjis_with_wrong_charset.txt'))
-    stub_request(:head, 'http://example.com/koi8-r').to_return(status: 200, headers: { 'Content-Type' => 'text/html' })
     stub_request(:get, 'http://example.com/koi8-r').to_return(request_fixture('koi8-r.txt'))
-    stub_request(:head, 'http://example.com/日本語').to_return(status: 200, headers: { 'Content-Type' => 'text/html' })
     stub_request(:get, 'http://example.com/日本語').to_return(request_fixture('sjis.txt'))
-    stub_request(:head, 'https://github.com/qbi/WannaCry').to_return(status: 404)
-    stub_request(:head, 'http://example.com/test-').to_return(status: 200, headers: { 'Content-Type' => 'text/html' })
+    stub_request(:get, 'https://github.com/qbi/WannaCry').to_return(status: 404)
     stub_request(:get, 'http://example.com/test-').to_return(request_fixture('idn.txt'))
-    stub_request(:head, 'http://example.com/windows-1251').to_return(status: 200, headers: { 'Content-Type' => 'text/html' })
     stub_request(:get, 'http://example.com/windows-1251').to_return(request_fixture('windows-1251.txt'))
 
     subject.call(status)
@@ -90,11 +83,11 @@ RSpec.describe FetchLinkCardService, type: :service do
     let(:status) { Fabricate(:status, account: Fabricate(:account, domain: 'example.com'), text: 'Habt ihr ein paar gute Links zu #<span class="tag"><a href="https://quitter.se/tag/wannacry" target="_blank" rel="tag noopener" title="https://quitter.se/tag/wannacry">Wannacry</a></span> herumfliegen?   Ich will mal unter <br> <a href="https://github.com/qbi/WannaCry" target="_blank" rel="noopener" title="https://github.com/qbi/WannaCry">https://github.com/qbi/WannaCry</a> was sammeln. !<a href="http://sn.jonkman.ca/group/416/id" target="_blank" rel="noopener" title="http://sn.jonkman.ca/group/416/id">security</a>&nbsp;') }
 
     it 'parses out URLs' do
-      expect(a_request(:head, 'https://github.com/qbi/WannaCry')).to have_been_made.at_least_once
+      expect(a_request(:get, 'https://github.com/qbi/WannaCry')).to have_been_made.at_least_once
     end
 
     it 'ignores URLs to hashtags' do
-      expect(a_request(:head, 'https://quitter.se/tag/wannacry')).to_not have_been_made
+      expect(a_request(:get, 'https://quitter.se/tag/wannacry')).to_not have_been_made
     end
   end
 end
-- 
cgit 


From 3a4d994c40962a9abe45565a34e3d7a3ca1ccd49 Mon Sep 17 00:00:00 2001
From: ThibG <thib@sitedethib.com>
Date: Tue, 1 Oct 2019 15:10:00 +0200
Subject: Fix BootstrapTimelineService crashing when bootstrapped accounts are
 invalid (#12037)

* Add test to handle suspended and missing users in BootstrapTimelineService

* Fix BootstrapTimelineService crashing when bootstrapped accounts are invalid
---
 app/services/bootstrap_timeline_service.rb       | 8 +++++++-
 spec/services/bootstrap_timeline_service_spec.rb | 7 ++++++-
 2 files changed, 13 insertions(+), 2 deletions(-)

(limited to 'spec')

diff --git a/app/services/bootstrap_timeline_service.rb b/app/services/bootstrap_timeline_service.rb
index db2c83e5d..c489601c1 100644
--- a/app/services/bootstrap_timeline_service.rb
+++ b/app/services/bootstrap_timeline_service.rb
@@ -17,7 +17,11 @@ class BootstrapTimelineService < BaseService
 
   def autofollow_bootstrap_timeline_accounts!
     bootstrap_timeline_accounts.each do |target_account|
-      FollowService.new.call(@source_account, target_account)
+      begin
+        FollowService.new.call(@source_account, target_account)
+      rescue ActiveRecord::RecordNotFound, Mastodon::NotPermittedError
+        nil
+      end
     end
   end
 
@@ -40,7 +44,9 @@ class BootstrapTimelineService < BaseService
 
   def local_unlocked_accounts(usernames)
     Account.local
+           .without_suspended
            .where(username: usernames)
            .where(locked: false)
+           .where(moved_to_account_id: nil)
   end
 end
diff --git a/spec/services/bootstrap_timeline_service_spec.rb b/spec/services/bootstrap_timeline_service_spec.rb
index a765de791..a28d2407c 100644
--- a/spec/services/bootstrap_timeline_service_spec.rb
+++ b/spec/services/bootstrap_timeline_service_spec.rb
@@ -22,9 +22,10 @@ RSpec.describe BootstrapTimelineService, type: :service do
     context 'when setting is set' do
       let!(:alice) { Fabricate(:account, username: 'alice') }
       let!(:bob)   { Fabricate(:account, username: 'bob') }
+      let!(:eve)   { Fabricate(:account, username: 'eve', suspended: true) }
 
       before do
-        Setting.bootstrap_timeline_accounts = 'alice, bob'
+        Setting.bootstrap_timeline_accounts = 'alice, @bob, eve, unknown'
         subject.call(source_account)
       end
 
@@ -32,6 +33,10 @@ RSpec.describe BootstrapTimelineService, type: :service do
         expect(source_account.following?(alice)).to be true
         expect(source_account.following?(bob)).to be true
       end
+
+      it 'does not follow suspended account' do
+        expect(source_account.following?(eve)).to be false
+      end
     end
   end
 end
-- 
cgit 


From 62f60e86c281ae1cd0356a7630dbb76069e2ffde Mon Sep 17 00:00:00 2001
From: Eugen Rochko <eugen@zeonfederated.com>
Date: Wed, 2 Oct 2019 04:59:37 +0200
Subject: Fix account counters being overwritten by parallel writes (#12045)

---
 app/models/account_stat.rb                         | 22 +++++++++
 ...1001213028_add_lock_version_to_account_stats.rb | 15 ++++++
 db/schema.rb                                       |  3 +-
 spec/models/account_stat_spec.rb                   | 53 ++++++++++++++++++++++
 4 files changed, 92 insertions(+), 1 deletion(-)
 create mode 100644 db/migrate/20191001213028_add_lock_version_to_account_stats.rb

(limited to 'spec')

diff --git a/app/models/account_stat.rb b/app/models/account_stat.rb
index 1351f7d8a..c84e4217c 100644
--- a/app/models/account_stat.rb
+++ b/app/models/account_stat.rb
@@ -11,6 +11,7 @@
 #  created_at      :datetime         not null
 #  updated_at      :datetime         not null
 #  last_status_at  :datetime
+#  lock_version    :integer          default(0), not null
 #
 
 class AccountStat < ApplicationRecord
@@ -20,10 +21,26 @@ class AccountStat < ApplicationRecord
 
   def increment_count!(key)
     update(attributes_for_increment(key))
+  rescue ActiveRecord::StaleObjectError
+    begin
+      reload_with_id
+    rescue ActiveRecord::RecordNotFound
+      # Nothing to do
+    else
+      retry
+    end
   end
 
   def decrement_count!(key)
     update(key => [public_send(key) - 1, 0].max)
+  rescue ActiveRecord::StaleObjectError
+    begin
+      reload_with_id
+    rescue ActiveRecord::RecordNotFound
+      # Nothing to do
+    else
+      retry
+    end
   end
 
   private
@@ -33,4 +50,9 @@ class AccountStat < ApplicationRecord
     attrs[:last_status_at] = Time.now.utc if key == :statuses_count
     attrs
   end
+
+  def reload_with_id
+    self.id = find_by!(account: account).id if new_record?
+    reload
+  end
 end
diff --git a/db/migrate/20191001213028_add_lock_version_to_account_stats.rb b/db/migrate/20191001213028_add_lock_version_to_account_stats.rb
new file mode 100644
index 000000000..47f37cca2
--- /dev/null
+++ b/db/migrate/20191001213028_add_lock_version_to_account_stats.rb
@@ -0,0 +1,15 @@
+require Rails.root.join('lib', 'mastodon', 'migration_helpers')
+
+class AddLockVersionToAccountStats < ActiveRecord::Migration[5.2]
+  include Mastodon::MigrationHelpers
+
+  disable_ddl_transaction!
+
+  def up
+    safety_assured { add_column_with_default :account_stats, :lock_version, :integer, allow_null: false, default: 0 }
+  end
+
+  def down
+    remove_column :account_stats, :lock_version
+  end
+end
diff --git a/db/schema.rb b/db/schema.rb
index 557b777e0..2d516965c 100644
--- a/db/schema.rb
+++ b/db/schema.rb
@@ -10,7 +10,7 @@
 #
 # It's strongly recommended that you check this file into your version control system.
 
-ActiveRecord::Schema.define(version: 2019_09_27_232842) do
+ActiveRecord::Schema.define(version: 2019_10_01_213028) do
 
   # These are extensions that must be enabled in order to support this database
   enable_extension "plpgsql"
@@ -97,6 +97,7 @@ ActiveRecord::Schema.define(version: 2019_09_27_232842) do
     t.datetime "created_at", null: false
     t.datetime "updated_at", null: false
     t.datetime "last_status_at"
+    t.integer "lock_version", default: 0, null: false
     t.index ["account_id"], name: "index_account_stats_on_account_id", unique: true
   end
 
diff --git a/spec/models/account_stat_spec.rb b/spec/models/account_stat_spec.rb
index a94185109..8adc0d1d6 100644
--- a/spec/models/account_stat_spec.rb
+++ b/spec/models/account_stat_spec.rb
@@ -1,4 +1,57 @@
 require 'rails_helper'
 
 RSpec.describe AccountStat, type: :model do
+  describe '#increment_count!' do
+    it 'increments the count' do
+      account_stat = AccountStat.create(account: Fabricate(:account))
+      expect(account_stat.followers_count).to eq 0
+      account_stat.increment_count!(:followers_count)
+      expect(account_stat.followers_count).to eq 1
+    end
+
+    it 'increments the count in multi-threaded an environment' do
+      account_stat   = AccountStat.create(account: Fabricate(:account), statuses_count: 0)
+      increment_by   = 15
+      wait_for_start = true
+
+      threads = Array.new(increment_by) do
+        Thread.new do
+          true while wait_for_start
+          AccountStat.find(account_stat.id).increment_count!(:statuses_count)
+        end
+      end
+
+      wait_for_start = false
+      threads.each(&:join)
+
+      expect(account_stat.reload.statuses_count).to eq increment_by
+    end
+  end
+
+  describe '#decrement_count!' do
+    it 'decrements the count' do
+      account_stat = AccountStat.create(account: Fabricate(:account), followers_count: 15)
+      expect(account_stat.followers_count).to eq 15
+      account_stat.decrement_count!(:followers_count)
+      expect(account_stat.followers_count).to eq 14
+    end
+
+    it 'decrements the count in multi-threaded an environment' do
+      account_stat   = AccountStat.create(account: Fabricate(:account), statuses_count: 15)
+      decrement_by   = 10
+      wait_for_start = true
+
+      threads = Array.new(decrement_by) do
+        Thread.new do
+          true while wait_for_start
+          AccountStat.find(account_stat.id).decrement_count!(:statuses_count)
+        end
+      end
+
+      wait_for_start = false
+      threads.each(&:join)
+
+      expect(account_stat.reload.statuses_count).to eq 5
+    end
+  end
 end
-- 
cgit 


From eb83d6256e6dc5d9c93dcb2b0a55e1061fee735f Mon Sep 17 00:00:00 2001
From: Eugen Rochko <eugen@zeonfederated.com>
Date: Thu, 3 Oct 2019 17:50:59 +0200
Subject: Add `reason` param to `POST /api/v1/accounts` REST API (#12064)

For approval-required registrations mode
---
 app/controllers/api/v1/accounts_controller.rb | 2 +-
 app/services/app_sign_up_service.rb           | 7 ++++---
 spec/services/app_sign_up_service_spec.rb     | 9 +++++++++
 3 files changed, 14 insertions(+), 4 deletions(-)

(limited to 'spec')

diff --git a/app/controllers/api/v1/accounts_controller.rb b/app/controllers/api/v1/accounts_controller.rb
index c12e1c12e..d68d2715f 100644
--- a/app/controllers/api/v1/accounts_controller.rb
+++ b/app/controllers/api/v1/accounts_controller.rb
@@ -78,7 +78,7 @@ class Api::V1::AccountsController < Api::BaseController
   end
 
   def account_params
-    params.permit(:username, :email, :password, :agreement, :locale)
+    params.permit(:username, :email, :password, :agreement, :locale, :reason)
   end
 
   def check_enabled_registrations
diff --git a/app/services/app_sign_up_service.rb b/app/services/app_sign_up_service.rb
index 6dee9cd81..c9739c77d 100644
--- a/app/services/app_sign_up_service.rb
+++ b/app/services/app_sign_up_service.rb
@@ -4,9 +4,10 @@ class AppSignUpService < BaseService
   def call(app, params)
     return unless allowed_registrations?
 
-    user_params    = params.slice(:email, :password, :agreement, :locale)
-    account_params = params.slice(:username)
-    user           = User.create!(user_params.merge(created_by_application: app, password_confirmation: user_params[:password], account_attributes: account_params))
+    user_params           = params.slice(:email, :password, :agreement, :locale)
+    account_params        = params.slice(:username)
+    invite_request_params = { text: params[:reason] }
+    user                  = User.create!(user_params.merge(created_by_application: app, password_confirmation: user_params[:password], account_attributes: account_params, invite_request_attributes: invite_request_params))
 
     Doorkeeper::AccessToken.create!(application: app,
                                     resource_owner_id: user.id,
diff --git a/spec/services/app_sign_up_service_spec.rb b/spec/services/app_sign_up_service_spec.rb
index 7948bb53b..e7c7f3ba1 100644
--- a/spec/services/app_sign_up_service_spec.rb
+++ b/spec/services/app_sign_up_service_spec.rb
@@ -38,6 +38,15 @@ RSpec.describe AppSignUpService, type: :service do
       user = User.find_by(id: access_token.resource_owner_id)
       expect(user).to_not be_nil
       expect(user.account).to_not be_nil
+      expect(user.invite_request).to be_nil
+    end
+
+    it 'creates an account with invite request text' do
+      access_token = subject.call(app, good_params.merge(reason: 'Foo bar'))
+      expect(access_token).to_not be_nil
+      user = User.find_by(id: access_token.resource_owner_id)
+      expect(user).to_not be_nil
+      expect(user.invite_request&.text).to eq 'Foo bar'
     end
   end
 end
-- 
cgit 


From f665901e3c0930fb8b3741f6bc6f6a15dd0343f6 Mon Sep 17 00:00:00 2001
From: Eugen Rochko <eugen@zeonfederated.com>
Date: Sun, 6 Oct 2019 22:11:17 +0200
Subject: Fix performance of home feed regeneration (#12084)

Fetching statuses from all followed accounts at once takes too long
within Postgres. Fetching them one by one and merging in Ruby
could be a lot less resource-intensive

Because the query for dynamically fetching the home timeline is so
heavy, we can no longer offer it when the home timeline is missing
---
 .../api/v1/timelines/home_controller.rb            |   6 +-
 app/javascript/mastodon/actions/timelines.js       |   2 +-
 .../mastodon/components/missing_indicator.js       |  23 +++--
 .../mastodon/components/regeneration_indicator.js  |  18 ++++
 app/javascript/mastodon/components/status_list.js  |  15 +--
 .../mastodon/features/generic_not_found/index.js   |   2 +-
 app/javascript/styles/mastodon/components.scss     |  30 ++----
 app/javascript/styles/mastodon/introduction.scss   |   3 +-
 app/lib/feed_manager.rb                            | 106 +++++++++++++++------
 app/models/home_feed.rb                            |  16 +---
 app/models/status.rb                               |   4 -
 spec/models/home_feed_spec.rb                      |   5 +-
 spec/models/status_spec.rb                         |  43 ---------
 13 files changed, 132 insertions(+), 141 deletions(-)
 create mode 100644 app/javascript/mastodon/components/regeneration_indicator.js

(limited to 'spec')

diff --git a/app/controllers/api/v1/timelines/home_controller.rb b/app/controllers/api/v1/timelines/home_controller.rb
index fcd0757f1..ff5ede138 100644
--- a/app/controllers/api/v1/timelines/home_controller.rb
+++ b/app/controllers/api/v1/timelines/home_controller.rb
@@ -13,7 +13,7 @@ class Api::V1::Timelines::HomeController < Api::BaseController
     render json: @statuses,
            each_serializer: REST::StatusSerializer,
            relationships: StatusRelationshipsPresenter.new(@statuses, current_user&.account_id),
-           status: regeneration_in_progress? ? 206 : 200
+           status: account_home_feed.regenerating? ? 206 : 200
   end
 
   private
@@ -62,8 +62,4 @@ class Api::V1::Timelines::HomeController < Api::BaseController
   def pagination_since_id
     @statuses.first.id
   end
-
-  def regeneration_in_progress?
-    Redis.current.exists("account:#{current_account.id}:regeneration")
-  end
 end
diff --git a/app/javascript/mastodon/actions/timelines.js b/app/javascript/mastodon/actions/timelines.js
index 7eeba2aa7..bc2ac5e82 100644
--- a/app/javascript/mastodon/actions/timelines.js
+++ b/app/javascript/mastodon/actions/timelines.js
@@ -97,7 +97,7 @@ export function expandTimeline(timelineId, path, params = {}, done = noOp) {
     api(getState).get(path, { params }).then(response => {
       const next = getLinks(response).refs.find(link => link.rel === 'next');
       dispatch(importFetchedStatuses(response.data));
-      dispatch(expandTimelineSuccess(timelineId, response.data, next ? next.uri : null, response.code === 206, isLoadingRecent, isLoadingMore, isLoadingRecent && preferPendingItems));
+      dispatch(expandTimelineSuccess(timelineId, response.data, next ? next.uri : null, response.status === 206, isLoadingRecent, isLoadingMore, isLoadingRecent && preferPendingItems));
       done();
     }).catch(error => {
       dispatch(expandTimelineFail(timelineId, error, isLoadingMore));
diff --git a/app/javascript/mastodon/components/missing_indicator.js b/app/javascript/mastodon/components/missing_indicator.js
index 70d8c3b98..7b0101bab 100644
--- a/app/javascript/mastodon/components/missing_indicator.js
+++ b/app/javascript/mastodon/components/missing_indicator.js
@@ -1,17 +1,24 @@
 import React from 'react';
+import PropTypes from 'prop-types';
 import { FormattedMessage } from 'react-intl';
+import illustration from 'mastodon/../images/elephant_ui_disappointed.svg';
+import classNames from 'classnames';
 
-const MissingIndicator = () => (
-  <div className='regeneration-indicator missing-indicator'>
-    <div>
-      <div className='regeneration-indicator__figure' />
+const MissingIndicator = ({ fullPage }) => (
+  <div className={classNames('regeneration-indicator', { 'regeneration-indicator--without-header': fullPage })}>
+    <div className='regeneration-indicator__figure'>
+      <img src={illustration} alt='' />
+    </div>
 
-      <div className='regeneration-indicator__label'>
-        <FormattedMessage id='missing_indicator.label' tagName='strong' defaultMessage='Not found' />
-        <FormattedMessage id='missing_indicator.sublabel' defaultMessage='This resource could not be found' />
-      </div>
+    <div className='regeneration-indicator__label'>
+      <FormattedMessage id='missing_indicator.label' tagName='strong' defaultMessage='Not found' />
+      <FormattedMessage id='missing_indicator.sublabel' defaultMessage='This resource could not be found' />
     </div>
   </div>
 );
 
+MissingIndicator.propTypes = {
+  fullPage: PropTypes.bool,
+};
+
 export default MissingIndicator;
diff --git a/app/javascript/mastodon/components/regeneration_indicator.js b/app/javascript/mastodon/components/regeneration_indicator.js
new file mode 100644
index 000000000..faf88c6b5
--- /dev/null
+++ b/app/javascript/mastodon/components/regeneration_indicator.js
@@ -0,0 +1,18 @@
+import React from 'react';
+import { FormattedMessage } from 'react-intl';
+import illustration from 'mastodon/../images/elephant_ui_working.svg';
+
+const MissingIndicator = () => (
+  <div className='regeneration-indicator'>
+    <div className='regeneration-indicator__figure'>
+      <img src={illustration} alt='' />
+    </div>
+
+    <div className='regeneration-indicator__label'>
+      <FormattedMessage id='regeneration_indicator.label' tagName='strong' defaultMessage='Loading&hellip;' />
+      <FormattedMessage id='regeneration_indicator.sublabel' defaultMessage='Your home feed is being prepared!' />
+    </div>
+  </div>
+);
+
+export default MissingIndicator;
diff --git a/app/javascript/mastodon/components/status_list.js b/app/javascript/mastodon/components/status_list.js
index 745e6422d..e1b370c91 100644
--- a/app/javascript/mastodon/components/status_list.js
+++ b/app/javascript/mastodon/components/status_list.js
@@ -1,12 +1,12 @@
 import { debounce } from 'lodash';
 import React from 'react';
-import { FormattedMessage } from 'react-intl';
 import ImmutablePropTypes from 'react-immutable-proptypes';
 import PropTypes from 'prop-types';
 import StatusContainer from '../containers/status_container';
 import ImmutablePureComponent from 'react-immutable-pure-component';
 import LoadGap from './load_gap';
 import ScrollableList from './scrollable_list';
+import RegenerationIndicator from 'mastodon/components/regeneration_indicator';
 
 export default class StatusList extends ImmutablePureComponent {
 
@@ -81,18 +81,7 @@ export default class StatusList extends ImmutablePureComponent {
     const { isLoading, isPartial } = other;
 
     if (isPartial) {
-      return (
-        <div className='regeneration-indicator'>
-          <div>
-            <div className='regeneration-indicator__figure' />
-
-            <div className='regeneration-indicator__label'>
-              <FormattedMessage id='regeneration_indicator.label' tagName='strong' defaultMessage='Loading&hellip;' />
-              <FormattedMessage id='regeneration_indicator.sublabel' defaultMessage='Your home feed is being prepared!' />
-            </div>
-          </div>
-        </div>
-      );
+      return <RegenerationIndicator />;
     }
 
     let scrollableContent = (isLoading || statusIds.size > 0) ? (
diff --git a/app/javascript/mastodon/features/generic_not_found/index.js b/app/javascript/mastodon/features/generic_not_found/index.js
index 0290be47f..41cd61a5f 100644
--- a/app/javascript/mastodon/features/generic_not_found/index.js
+++ b/app/javascript/mastodon/features/generic_not_found/index.js
@@ -4,7 +4,7 @@ import MissingIndicator from '../../components/missing_indicator';
 
 const GenericNotFound = () => (
   <Column>
-    <MissingIndicator />
+    <MissingIndicator fullPage />
   </Column>
 );
 
diff --git a/app/javascript/styles/mastodon/components.scss b/app/javascript/styles/mastodon/components.scss
index a6675b8ed..d9182ade9 100644
--- a/app/javascript/styles/mastodon/components.scss
+++ b/app/javascript/styles/mastodon/components.scss
@@ -3127,37 +3127,27 @@ a.status-card.compact:hover {
   cursor: default;
   display: flex;
   flex: 1 1 auto;
+  flex-direction: column;
   align-items: center;
   justify-content: center;
   padding: 20px;
 
-  & > div {
-    width: 100%;
-    background: transparent;
-    padding-top: 0;
-  }
-
   &__figure {
-    background: url('../images/elephant_ui_working.svg') no-repeat center 0;
-    width: 100%;
-    height: 160px;
-    background-size: contain;
-    position: absolute;
-    top: 50%;
-    left: 50%;
-    transform: translate(-50%, -50%);
+    &,
+    img {
+      display: block;
+      width: auto;
+      height: 160px;
+      margin: 0;
+    }
   }
 
-  &.missing-indicator {
+  &--without-header {
     padding-top: 20px + 48px;
-
-    .regeneration-indicator__figure {
-      background-image: url('../images/elephant_ui_disappointed.svg');
-    }
   }
 
   &__label {
-    margin-top: 200px;
+    margin-top: 30px;
 
     strong {
       display: block;
diff --git a/app/javascript/styles/mastodon/introduction.scss b/app/javascript/styles/mastodon/introduction.scss
index 222d8f60e..b44ae7306 100644
--- a/app/javascript/styles/mastodon/introduction.scss
+++ b/app/javascript/styles/mastodon/introduction.scss
@@ -3,9 +3,10 @@
   flex-direction: column;
   justify-content: center;
   align-items: center;
+  height: 100vh;
+  background: $ui-base-color;
 
   @media screen and (max-width: 920px) {
-    background: darken($ui-base-color, 8%);
     display: block !important;
   }
 
diff --git a/app/lib/feed_manager.rb b/app/lib/feed_manager.rb
index 871ec5c19..d8b486b60 100644
--- a/app/lib/feed_manager.rb
+++ b/app/lib/feed_manager.rb
@@ -19,7 +19,7 @@ class FeedManager
 
   def filter?(timeline_type, status, receiver_id)
     if timeline_type == :home
-      filter_from_home?(status, receiver_id)
+      filter_from_home?(status, receiver_id, build_crutches(receiver_id, [status]))
     elsif timeline_type == :mentions
       filter_from_mentions?(status, receiver_id)
     else
@@ -29,6 +29,7 @@ class FeedManager
 
   def push_to_home(account, status)
     return false unless add_to_feed(:home, account.id, status, account.user&.aggregates_reblogs?)
+
     trim(:home, account.id)
     PushUpdateWorker.perform_async(account.id, status.id, "timeline:#{account.id}") if push_update_required?("timeline:#{account.id}")
     true
@@ -36,6 +37,7 @@ class FeedManager
 
   def unpush_from_home(account, status)
     return false unless remove_from_feed(:home, account.id, status, account.user&.aggregates_reblogs?)
+
     redis.publish("timeline:#{account.id}", Oj.dump(event: :delete, payload: status.id.to_s))
     true
   end
@@ -46,7 +48,9 @@ class FeedManager
       should_filter &&= !ListAccount.where(list_id: list.id, account_id: status.in_reply_to_account_id).exists?
       return false if should_filter
     end
+
     return false unless add_to_feed(:list, list.id, status, list.account.user&.aggregates_reblogs?)
+
     trim(:list, list.id)
     PushUpdateWorker.perform_async(list.account_id, status.id, "timeline:list:#{list.id}") if push_update_required?("timeline:list:#{list.id}")
     true
@@ -54,6 +58,7 @@ class FeedManager
 
   def unpush_from_list(list, status)
     return false unless remove_from_feed(:list, list.id, status, list.account.user&.aggregates_reblogs?)
+
     redis.publish("timeline:list:#{list.id}", Oj.dump(event: :delete, payload: status.id.to_s))
     true
   end
@@ -85,16 +90,21 @@ class FeedManager
 
   def merge_into_timeline(from_account, into_account)
     timeline_key = key(:home, into_account.id)
-    query        = from_account.statuses.limit(FeedManager::MAX_ITEMS / 4)
+    aggregate    = into_account.user&.aggregates_reblogs?
+    query        = from_account.statuses.where(visibility: [:public, :unlisted, :private]).includes(:preloadable_poll, reblog: :account).limit(FeedManager::MAX_ITEMS / 4)
 
     if redis.zcard(timeline_key) >= FeedManager::MAX_ITEMS / 4
-      oldest_home_score = redis.zrange(timeline_key, 0, 0, with_scores: true)&.first&.last&.to_i || 0
+      oldest_home_score = redis.zrange(timeline_key, 0, 0, with_scores: true).first.last.to_i
       query = query.where('id > ?', oldest_home_score)
     end
 
-    query.each do |status|
-      next if status.direct_visibility? || status.limited_visibility? || filter?(:home, status, into_account)
-      add_to_feed(:home, into_account.id, status, into_account.user&.aggregates_reblogs?)
+    statuses = query.to_a
+    crutches = build_crutches(into_account.id, statuses)
+
+    statuses.each do |status|
+      next if filter_from_home?(status, into_account, crutches)
+
+      add_to_feed(:home, into_account.id, status, aggregate)
     end
 
     trim(:home, into_account.id)
@@ -120,24 +130,35 @@ class FeedManager
   end
 
   def populate_feed(account)
-    added  = 0
-    limit  = FeedManager::MAX_ITEMS / 2
-    max_id = nil
+    limit        = FeedManager::MAX_ITEMS / 2
+    aggregate    = account.user&.aggregates_reblogs?
+    timeline_key = key(:home, account.id)
 
-    loop do
-      statuses = Status.as_home_timeline(account)
-                       .paginate_by_max_id(limit, max_id)
+    account.statuses.where.not(visibility: :direct).limit(limit).each do |status|
+      add_to_feed(:home, account.id, status, aggregate)
+    end
 
-      break if statuses.empty?
+    account.following.includes(:account_stat).find_each do |target_account|
+      if redis.zcard(timeline_key) >= limit
+        oldest_home_score = redis.zrange(timeline_key, 0, 0, with_scores: true).first.last.to_i
+        last_status_score = Mastodon::Snowflake.id_at(account.last_status_at)
 
-      statuses.each do |status|
-        next if filter_from_home?(status, account)
-        added += 1 if add_to_feed(:home, account.id, status, account.user&.aggregates_reblogs?)
+        # If the feed is full and this account has not posted more recently
+        # than the last item on the feed, then we can skip the whole account
+        # because none of its statuses would stay on the feed anyway
+        next if last_status_score < oldest_home_score
       end
 
-      break unless added.zero?
+      statuses = target_account.statuses.where(visibility: [:public, :unlisted, :private]).includes(:preloadable_poll, reblog: :account).limit(limit)
+      crutches = build_crutches(account.id, statuses)
+
+      statuses.each do |status|
+        next if filter_from_home?(status, account, crutches)
+
+        add_to_feed(:home, account.id, status, aggregate)
+      end
 
-      max_id = statuses.last.id
+      trim(:home, account.id)
     end
   end
 
@@ -152,31 +173,33 @@ class FeedManager
       (context == :home ? Mute.where(account_id: receiver_id, target_account_id: account_ids).any? : Mute.where(account_id: receiver_id, target_account_id: account_ids, hide_notifications: true).any?)
   end
 
-  def filter_from_home?(status, receiver_id)
+  def filter_from_home?(status, receiver_id, crutches)
     return false if receiver_id == status.account_id
     return true  if status.reply? && (status.in_reply_to_id.nil? || status.in_reply_to_account_id.nil?)
     return true  if phrase_filtered?(status, receiver_id, :home)
 
-    check_for_blocks = status.active_mentions.pluck(:account_id)
+    check_for_blocks = crutches[:active_mentions][status.id] || []
     check_for_blocks.concat([status.account_id])
 
     if status.reblog?
       check_for_blocks.concat([status.reblog.account_id])
-      check_for_blocks.concat(status.reblog.active_mentions.pluck(:account_id))
+      check_for_blocks.concat(crutches[:active_mentions][status.reblog_of_id] || [])
     end
 
-    return true if blocks_or_mutes?(receiver_id, check_for_blocks, :home)
+    return true if check_for_blocks.any? { |target_account_id| crutches[:blocking][target_account_id] || crutches[:muting][target_account_id] }
 
     if status.reply? && !status.in_reply_to_account_id.nil?                                                                      # Filter out if it's a reply
-      should_filter   = !Follow.where(account_id: receiver_id, target_account_id: status.in_reply_to_account_id).exists?         # and I'm not following the person it's a reply to
+      should_filter   = !crutches[:following][status.in_reply_to_account_id]                                                     # and I'm not following the person it's a reply to
       should_filter &&= receiver_id != status.in_reply_to_account_id                                                             # and it's not a reply to me
       should_filter &&= status.account_id != status.in_reply_to_account_id                                                       # and it's not a self-reply
-      return should_filter
+
+      return !!should_filter
     elsif status.reblog?                                                                                                         # Filter out a reblog
-      should_filter   = Follow.where(account_id: receiver_id, target_account_id: status.account_id, show_reblogs: false).exists? # if the reblogger's reblogs are suppressed
-      should_filter ||= Block.where(account_id: status.reblog.account_id, target_account_id: receiver_id).exists?                # or if the author of the reblogged status is blocking me
-      should_filter ||= AccountDomainBlock.where(account_id: receiver_id, domain: status.reblog.account.domain).exists?          # or the author's domain is blocked
-      return should_filter
+      should_filter   = crutches[:hiding_reblogs][status.account_id]                                                             # if the reblogger's reblogs are suppressed
+      should_filter ||= crutches[:blocked_by][status.reblog.account_id]                                                          # or if the author of the reblogged status is blocking me
+      should_filter ||= crutches[:domain_blocking][status.reblog.account.domain]                                                 # or the author's domain is blocked
+
+      return !!should_filter
     end
 
     false
@@ -308,4 +331,31 @@ class FeedManager
 
     redis.zrem(timeline_key, status.id)
   end
+
+  def build_crutches(receiver_id, statuses)
+    crutches = {}
+
+    crutches[:active_mentions] = Mention.active.where(status_id: statuses.flat_map { |s| [s.id, s.reblog_of_id] }.compact).pluck(:status_id, :account_id).each_with_object({}) { |(id, account_id), mapping| (mapping[id] ||= []).push(account_id) }
+
+    check_for_blocks = statuses.flat_map do |s|
+      arr = crutches[:active_mentions][s.id] || []
+      arr.concat([s.account_id])
+
+      if s.reblog?
+        arr.concat([s.reblog.account_id])
+        arr.concat(crutches[:active_mentions][s.reblog_of_id] || [])
+      end
+
+      arr
+    end
+
+    crutches[:following]       = Follow.where(account_id: receiver_id, target_account_id: statuses.map(&:in_reply_to_account_id).compact).pluck(:target_account_id).each_with_object({}) { |id, mapping| mapping[id] = true }
+    crutches[:hiding_reblogs]  = Follow.where(account_id: receiver_id, target_account_id: statuses.map { |s| s.account_id if s.reblog? }.compact, show_reblogs: false).pluck(:target_account_id).each_with_object({}) { |id, mapping| mapping[id] = true }
+    crutches[:blocking]        = Block.where(account_id: receiver_id, target_account_id: check_for_blocks).pluck(:target_account_id).each_with_object({}) { |id, mapping| mapping[id] = true }
+    crutches[:muting]          = Mute.where(account_id: receiver_id, target_account_id: check_for_blocks).pluck(:target_account_id).each_with_object({}) { |id, mapping| mapping[id] = true }
+    crutches[:domain_blocking] = AccountDomainBlock.where(account_id: receiver_id, domain: statuses.map { |s| s.reblog&.account&.domain }.compact).pluck(:domain).each_with_object({}) { |domain, mapping| mapping[domain] = true }
+    crutches[:blocked_by]      = Block.where(target_account_id: receiver_id, account_id: statuses.map { |s| s.reblog&.account_id }.compact).pluck(:account_id).each_with_object({}) { |id, mapping| mapping[id] = true }
+
+    crutches
+  end
 end
diff --git a/app/models/home_feed.rb b/app/models/home_feed.rb
index ba7564983..1fd506138 100644
--- a/app/models/home_feed.rb
+++ b/app/models/home_feed.rb
@@ -7,19 +7,7 @@ class HomeFeed < Feed
     @account = account
   end
 
-  def get(limit, max_id = nil, since_id = nil, min_id = nil)
-    if redis.exists("account:#{@account.id}:regeneration")
-      from_database(limit, max_id, since_id, min_id)
-    else
-      super
-    end
-  end
-
-  private
-
-  def from_database(limit, max_id, since_id, min_id)
-    Status.as_home_timeline(@account)
-          .paginate_by_id(limit, max_id: max_id, since_id: since_id, min_id: min_id)
-          .reject { |status| FeedManager.instance.filter?(:home, status, @account.id) }
+  def regenerating?
+    redis.exists("account:#{@id}:regeneration")
   end
 end
diff --git a/app/models/status.rb b/app/models/status.rb
index 3504ac370..0c01a5389 100644
--- a/app/models/status.rb
+++ b/app/models/status.rb
@@ -282,10 +282,6 @@ class Status < ApplicationRecord
       where(language: nil).or where(language: account.chosen_languages)
     end
 
-    def as_home_timeline(account)
-      where(account: [account] + account.following).where(visibility: [:public, :unlisted, :private])
-    end
-
     def as_public_timeline(account = nil, local_only = false)
       query = timeline_scope(local_only).without_replies
 
diff --git a/spec/models/home_feed_spec.rb b/spec/models/home_feed_spec.rb
index 3acb997f1..ee7a83960 100644
--- a/spec/models/home_feed_spec.rb
+++ b/spec/models/home_feed_spec.rb
@@ -34,11 +34,10 @@ RSpec.describe HomeFeed, type: :model do
         Redis.current.set("account:#{account.id}:regeneration", true)
       end
 
-      it 'gets statuses with ids in the range from database' do
+      it 'returns nothing' do
         results = subject.get(3)
 
-        expect(results.map(&:id)).to eq [10, 3, 2]
-        expect(results.first.attributes.keys).to include('id', 'updated_at')
+        expect(results.map(&:id)).to eq []
       end
     end
   end
diff --git a/spec/models/status_spec.rb b/spec/models/status_spec.rb
index a8c567414..51a10cd17 100644
--- a/spec/models/status_spec.rb
+++ b/spec/models/status_spec.rb
@@ -296,49 +296,6 @@ RSpec.describe Status, type: :model do
     end
   end
 
-  describe '.as_home_timeline' do
-    let(:account) { Fabricate(:account) }
-    let(:followed) { Fabricate(:account) }
-    let(:not_followed) { Fabricate(:account) }
-
-    before do
-      Fabricate(:follow, account: account, target_account: followed)
-
-      @self_status = Fabricate(:status, account: account, visibility: :public)
-      @self_direct_status = Fabricate(:status, account: account, visibility: :direct)
-      @followed_status = Fabricate(:status, account: followed, visibility: :public)
-      @followed_direct_status = Fabricate(:status, account: followed, visibility: :direct)
-      @not_followed_status = Fabricate(:status, account: not_followed, visibility: :public)
-
-      @results = Status.as_home_timeline(account)
-    end
-
-    it 'includes statuses from self' do
-      expect(@results).to include(@self_status)
-    end
-
-    it 'does not include direct statuses from self' do
-      expect(@results).to_not include(@self_direct_status)
-    end
-
-    it 'includes statuses from followed' do
-      expect(@results).to include(@followed_status)
-    end
-
-    it 'does not include direct statuses mentioning recipient from followed' do
-      Fabricate(:mention, account: account, status: @followed_direct_status)
-      expect(@results).to_not include(@followed_direct_status)
-    end
-
-    it 'does not include direct statuses not mentioning recipient from followed' do
-      expect(@results).not_to include(@followed_direct_status)
-    end
-
-    it 'does not include statuses from non-followed' do
-      expect(@results).not_to include(@not_followed_status)
-    end
-  end
-
   describe '.as_public_timeline' do
     it 'only includes statuses with public visibility' do
       public_status = Fabricate(:status, visibility: :public)
-- 
cgit 


From 354fdd317e9c495ed721013911bc5274d5e0e1f8 Mon Sep 17 00:00:00 2001
From: Eugen Rochko <eugen@zeonfederated.com>
Date: Wed, 9 Oct 2019 07:10:46 +0200
Subject: Fix attachment not being re-downloaded even if file is not stored
 (#12125)

Change the behaviour of remotable concern. Previously, it would skip
downloading an attachment if the stored remote URL is identical to
the new one. Now it would not be skipped if the attachment is not
actually currently stored by Paperclip.
---
 app/controllers/api/v1/streaming_controller.rb | 14 ++++++++++----
 app/models/account.rb                          |  7 +++----
 app/models/concerns/remotable.rb               |  2 +-
 config/initializers/paperclip.rb               | 19 +++++++++++++------
 spec/models/account_spec.rb                    |  4 ++--
 spec/models/concerns/remotable_spec.rb         | 13 ++++++++++++-
 6 files changed, 41 insertions(+), 18 deletions(-)

(limited to 'spec')

diff --git a/app/controllers/api/v1/streaming_controller.rb b/app/controllers/api/v1/streaming_controller.rb
index 66b812e76..ebb17608c 100644
--- a/app/controllers/api/v1/streaming_controller.rb
+++ b/app/controllers/api/v1/streaming_controller.rb
@@ -5,11 +5,17 @@ class Api::V1::StreamingController < Api::BaseController
 
   def index
     if Rails.configuration.x.streaming_api_base_url != request.host
-      uri = URI.parse(request.url)
-      uri.host = URI.parse(Rails.configuration.x.streaming_api_base_url).host
-      redirect_to uri.to_s, status: 301
+      redirect_to streaming_api_url, status: 301
     else
-      raise ActiveRecord::RecordNotFound
+      not_found
     end
   end
+
+  private
+
+  def streaming_api_url
+    Addressable::URI.parse(request.url).tap do |uri|
+      uri.host = Addressable::URI.parse(Rails.configuration.x.streaming_api_base_url).host
+    end.to_s
+  end
 end
diff --git a/app/models/account.rb b/app/models/account.rb
index 01d45e36c..2f43f337f 100644
--- a/app/models/account.rb
+++ b/app/models/account.rb
@@ -310,10 +310,9 @@ class Account < ApplicationRecord
   def save_with_optional_media!
     save!
   rescue ActiveRecord::RecordInvalid
-    self.avatar              = nil
-    self.header              = nil
-    self[:avatar_remote_url] = ''
-    self[:header_remote_url] = ''
+    self.avatar = nil
+    self.header = nil
+
     save!
   end
 
diff --git a/app/models/concerns/remotable.rb b/app/models/concerns/remotable.rb
index 082302619..b7a476c87 100644
--- a/app/models/concerns/remotable.rb
+++ b/app/models/concerns/remotable.rb
@@ -18,7 +18,7 @@ module Remotable
           return
         end
 
-        return if !%w(http https).include?(parsed_url.scheme) || parsed_url.host.blank? || self[attribute_name] == url
+        return if !%w(http https).include?(parsed_url.scheme) || parsed_url.host.blank? || (self[attribute_name] == url && send("#{attachment_name}_file_name").present?)
 
         begin
           Request.new(:get, url).perform do |response|
diff --git a/config/initializers/paperclip.rb b/config/initializers/paperclip.rb
index a0253f4bc..d3602e655 100644
--- a/config/initializers/paperclip.rb
+++ b/config/initializers/paperclip.rb
@@ -1,10 +1,11 @@
 # frozen_string_literal: true
 
-Paperclip.options[:read_timeout] = 60
-
 Paperclip.interpolates :filename do |attachment, style|
-  return attachment.original_filename if style == :original
-  [basename(attachment, style), extension(attachment, style)].delete_if(&:blank?).join('.')
+  if style == :original
+    attachment.original_filename
+  else
+    [basename(attachment, style), extension(attachment, style)].delete_if(&:blank?).join('.')
+  end
 end
 
 Paperclip::Attachment.default_options.merge!(
@@ -24,17 +25,21 @@ if ENV['S3_ENABLED'] == 'true'
     storage: :s3,
     s3_protocol: s3_protocol,
     s3_host_name: s3_hostname,
+
     s3_headers: {
       'X-Amz-Multipart-Threshold' => ENV.fetch('S3_MULTIPART_THRESHOLD') { 15.megabytes }.to_i,
       'Cache-Control' => 'public, max-age=315576000, immutable',
     },
+
     s3_permissions: ENV.fetch('S3_PERMISSION') { 'public-read' },
     s3_region: s3_region,
+
     s3_credentials: {
       bucket: ENV['S3_BUCKET'],
       access_key_id: ENV['AWS_ACCESS_KEY_ID'],
       secret_access_key: ENV['AWS_SECRET_ACCESS_KEY'],
     },
+
     s3_options: {
       signature_version: ENV.fetch('S3_SIGNATURE_VERSION') { 'v4' },
       http_open_timeout: 5,
@@ -49,6 +54,7 @@ if ENV['S3_ENABLED'] == 'true'
       endpoint: ENV['S3_ENDPOINT'],
       force_path_style: true
     )
+
     Paperclip::Attachment.default_options[:url] = ':s3_path_url'
   end
 
@@ -74,6 +80,7 @@ elsif ENV['SWIFT_ENABLED'] == 'true'
       openstack_region: ENV['SWIFT_REGION'],
       openstack_cache_ttl: ENV.fetch('SWIFT_CACHE_TTL') { 60 },
     },
+
     fog_directory: ENV['SWIFT_CONTAINER'],
     fog_host: ENV['SWIFT_OBJECT_URL'],
     fog_public: true
@@ -82,7 +89,7 @@ else
   Paperclip::Attachment.default_options.merge!(
     storage: :filesystem,
     use_timestamp: true,
-    path: (ENV['PAPERCLIP_ROOT_PATH'] || ':rails_root/public/system') + '/:class/:attachment/:id_partition/:style/:filename',
-    url: (ENV['PAPERCLIP_ROOT_URL'] || '/system') + '/:class/:attachment/:id_partition/:style/:filename',
+    path: ENV.fetch('PAPERCLIP_ROOT_PATH', ':rails_root/public/system') + '/:class/:attachment/:id_partition/:style/:filename',
+    url: ENV.fetch('PAPERCLIP_ROOT_URL', '/system') + '/:class/:attachment/:id_partition/:style/:filename',
   )
 end
diff --git a/spec/models/account_spec.rb b/spec/models/account_spec.rb
index 3eec464bd..b2f6234cb 100644
--- a/spec/models/account_spec.rb
+++ b/spec/models/account_spec.rb
@@ -126,8 +126,8 @@ RSpec.describe Account, type: :model do
       end
 
       it 'sets default avatar, header, avatar_remote_url, and header_remote_url' do
-        expect(account.avatar_remote_url).to eq ''
-        expect(account.header_remote_url).to eq ''
+        expect(account.avatar_remote_url).to eq 'https://remote.test/invalid_avatar'
+        expect(account.header_remote_url).to eq expectation.header_remote_url
         expect(account.avatar_file_name).to  eq nil
         expect(account.header_file_name).to  eq nil
       end
diff --git a/spec/models/concerns/remotable_spec.rb b/spec/models/concerns/remotable_spec.rb
index a4289cc45..99a60cbf6 100644
--- a/spec/models/concerns/remotable_spec.rb
+++ b/spec/models/concerns/remotable_spec.rb
@@ -18,6 +18,8 @@ RSpec.describe Remotable do
 
     def hoge=(arg); end
 
+    def hoge_file_name; end
+
     def hoge_file_name=(arg); end
 
     def has_attribute?(arg); end
@@ -109,12 +111,21 @@ RSpec.describe Remotable do
       end
 
       context 'foo[attribute_name] == url' do
-        it 'makes no request' do
+        it 'makes no request if file is saved' do
           allow(foo).to receive(:[]).with(attribute_name).and_return(url)
+          allow(foo).to receive(:hoge_file_name).and_return('foo.jpg')
 
           foo.hoge_remote_url = url
           expect(request).not_to have_been_requested
         end
+
+        it 'makes request if file is not saved' do
+          allow(foo).to receive(:[]).with(attribute_name).and_return(url)
+          allow(foo).to receive(:hoge_file_name).and_return(nil)
+
+          foo.hoge_remote_url = url
+          expect(request).to have_been_requested
+        end
       end
 
       context "scheme is https, parsed_url.host isn't empty, and foo[attribute_name] != url" do
-- 
cgit 


From b5f7e12817356b9b1795ab0187fe08d07f13a485 Mon Sep 17 00:00:00 2001
From: Eugen Rochko <eugen@zeonfederated.com>
Date: Wed, 9 Oct 2019 07:11:23 +0200
Subject: Remove auto-silence behaviour from spam check (#12117)

Fix #12113
---
 app/lib/spam_check.rb              |  7 +------
 app/models/account.rb              |  2 +-
 app/models/admin/account_action.rb | 12 ++++++++++++
 config/locales/en.yml              |  2 +-
 spec/lib/spam_check_spec.rb        |  4 ----
 5 files changed, 15 insertions(+), 12 deletions(-)

(limited to 'spec')

diff --git a/app/lib/spam_check.rb b/app/lib/spam_check.rb
index 441697364..235e44230 100644
--- a/app/lib/spam_check.rb
+++ b/app/lib/spam_check.rb
@@ -44,7 +44,6 @@ class SpamCheck
   end
 
   def flag!
-    auto_silence_account!
     auto_report_status!
   end
 
@@ -134,17 +133,13 @@ class SpamCheck
     text.gsub(/\s+/, ' ').strip
   end
 
-  def auto_silence_account!
-    @account.silence!
-  end
-
   def auto_report_status!
     status_ids = Status.where(visibility: %i(public unlisted)).where(id: matching_status_ids).pluck(:id) + [@status.id] if @status.distributable?
     ReportService.new.call(Account.representative, @account, status_ids: status_ids, comment: I18n.t('spam_check.spam_detected_and_silenced'))
   end
 
   def already_flagged?
-    @account.silenced?
+    @account.silenced? || @account.targeted_reports.unresolved.where(account_id: -99).exists?
   end
 
   def trusted?
diff --git a/app/models/account.rb b/app/models/account.rb
index 2f43f337f..05936def3 100644
--- a/app/models/account.rb
+++ b/app/models/account.rb
@@ -198,7 +198,7 @@ class Account < ApplicationRecord
   end
 
   def unsilence!
-    update!(silenced_at: nil, trust_level: trust_level == TRUST_LEVELS[:untrusted] ? TRUST_LEVELS[:trusted] : trust_level)
+    update!(silenced_at: nil)
   end
 
   def suspended?
diff --git a/app/models/admin/account_action.rb b/app/models/admin/account_action.rb
index b30a82369..e9da003a3 100644
--- a/app/models/admin/account_action.rb
+++ b/app/models/admin/account_action.rb
@@ -62,6 +62,8 @@ class Admin::AccountAction
 
   def process_action!
     case type
+    when 'none'
+      handle_resolve!
     when 'disable'
       handle_disable!
     when 'silence'
@@ -103,6 +105,16 @@ class Admin::AccountAction
     end
   end
 
+  def handle_resolve!
+    if with_report? && report.account_id == -99 && target_account.trust_level == Account::TRUST_LEVELS[:untrusted]
+      # This is an automated report and it is being dismissed, so it's
+      # a false positive, in which case update the account's trust level
+      # to prevent further spam checks
+
+      target_account.update(trust_level: Account::TRUST_LEVELS[:trusted])
+    end
+  end
+
   def handle_disable!
     authorize(target_account.user, :disable?)
     log_action(:disable, target_account.user)
diff --git a/config/locales/en.yml b/config/locales/en.yml
index 0e8ee6a76..1ffc99eb3 100644
--- a/config/locales/en.yml
+++ b/config/locales/en.yml
@@ -497,7 +497,7 @@ en:
         title: Custom terms of service
       site_title: Server name
       spam_check_enabled:
-        desc_html: Mastodon can auto-silence and auto-report accounts that send repeated unsolicited messages. There may be false positives.
+        desc_html: Mastodon can auto-report accounts that send repeated unsolicited messages. There may be false positives.
         title: Anti-spam automation
       thumbnail:
         desc_html: Used for previews via OpenGraph and API. 1200x630px recommended
diff --git a/spec/lib/spam_check_spec.rb b/spec/lib/spam_check_spec.rb
index 4cae46111..d4d66a499 100644
--- a/spec/lib/spam_check_spec.rb
+++ b/spec/lib/spam_check_spec.rb
@@ -181,10 +181,6 @@ RSpec.describe SpamCheck do
       described_class.new(status2).flag!
     end
 
-    it 'silences the account' do
-      expect(sender.silenced?).to be true
-    end
-
     it 'creates a report about the account' do
       expect(sender.targeted_reports.unresolved.count).to eq 1
     end
-- 
cgit 


From fccf83e1f2ecd4e23f7b1faee5330976d17da7b8 Mon Sep 17 00:00:00 2001
From: BSKY <git@bsky.moe>
Date: Fri, 25 Oct 2019 05:44:42 +0900
Subject: Add noopener and/or noreferrer (#12202)

---
 app/javascript/mastodon/components/attachment_list.js      |  4 ++--
 app/javascript/mastodon/components/dropdown_menu.js        |  2 +-
 app/javascript/mastodon/components/error_boundary.js       |  2 +-
 app/javascript/mastodon/components/media_gallery.js        |  3 ++-
 app/javascript/mastodon/components/status.js               |  4 ++--
 app/javascript/mastodon/components/status_content.js       |  2 +-
 .../mastodon/features/account/components/header.js         |  6 +++---
 .../features/account_gallery/components/media_item.js      | 14 +++++++-------
 app/javascript/mastodon/features/status/components/card.js |  6 +++---
 .../mastodon/features/status/components/detailed_status.js |  4 ++--
 .../mastodon/features/ui/components/actions_modal.js       |  4 ++--
 .../mastodon/features/ui/components/boost_modal.js         |  2 +-
 .../mastodon/features/ui/components/link_footer.js         |  2 +-
 app/lib/formatter.rb                                       |  2 +-
 app/lib/sanitize_config.rb                                 |  2 +-
 app/views/about/show.html.haml                             |  2 +-
 app/views/accounts/_moved.html.haml                        |  2 +-
 app/views/admin/reports/_status.html.haml                  |  2 +-
 app/views/admin/tags/show.html.haml                        |  2 +-
 app/views/application/_card.html.haml                      |  2 +-
 app/views/oauth/authorized_applications/index.html.haml    |  2 +-
 app/views/statuses/_detailed_status.html.haml              |  4 ++--
 app/views/statuses/_simple_status.html.haml                |  4 ++--
 spec/fixtures/xml/mastodon.atom                            |  4 ++--
 spec/lib/sanitize_config_spec.rb                           |  2 +-
 spec/services/fetch_link_card_service_spec.rb              |  2 +-
 spec/services/verify_link_service_spec.rb                  |  4 ++--
 27 files changed, 46 insertions(+), 45 deletions(-)

(limited to 'spec')

diff --git a/app/javascript/mastodon/components/attachment_list.js b/app/javascript/mastodon/components/attachment_list.js
index 5dfa1464c..ebd696583 100644
--- a/app/javascript/mastodon/components/attachment_list.js
+++ b/app/javascript/mastodon/components/attachment_list.js
@@ -25,7 +25,7 @@ export default class AttachmentList extends ImmutablePureComponent {
 
               return (
                 <li key={attachment.get('id')}>
-                  <a href={displayUrl} target='_blank' rel='noopener'><Icon id='link' /> {filename(displayUrl)}</a>
+                  <a href={displayUrl} target='_blank' rel='noopener noreferrer'><Icon id='link' /> {filename(displayUrl)}</a>
                 </li>
               );
             })}
@@ -46,7 +46,7 @@ export default class AttachmentList extends ImmutablePureComponent {
 
             return (
               <li key={attachment.get('id')}>
-                <a href={displayUrl} target='_blank' rel='noopener'>{filename(displayUrl)}</a>
+                <a href={displayUrl} target='_blank' rel='noopener noreferrer'>{filename(displayUrl)}</a>
               </li>
             );
           })}
diff --git a/app/javascript/mastodon/components/dropdown_menu.js b/app/javascript/mastodon/components/dropdown_menu.js
index d423378c1..a4f262285 100644
--- a/app/javascript/mastodon/components/dropdown_menu.js
+++ b/app/javascript/mastodon/components/dropdown_menu.js
@@ -143,7 +143,7 @@ class DropdownMenu extends React.PureComponent {
 
     return (
       <li className='dropdown-menu__item' key={`${text}-${i}`}>
-        <a href={href} target={target} data-method={method} rel='noopener' role='button' tabIndex='0' ref={i === 0 ? this.setFocusRef : null} onClick={this.handleClick} onKeyPress={this.handleItemKeyPress} data-index={i}>
+        <a href={href} target={target} data-method={method} rel='noopener noreferrer' role='button' tabIndex='0' ref={i === 0 ? this.setFocusRef : null} onClick={this.handleClick} onKeyPress={this.handleItemKeyPress} data-index={i}>
           {text}
         </a>
       </li>
diff --git a/app/javascript/mastodon/components/error_boundary.js b/app/javascript/mastodon/components/error_boundary.js
index 82543e118..800b1c270 100644
--- a/app/javascript/mastodon/components/error_boundary.js
+++ b/app/javascript/mastodon/components/error_boundary.js
@@ -58,7 +58,7 @@ export default class ErrorBoundary extends React.PureComponent {
         <div>
           <p className='error-boundary__error'><FormattedMessage id='error.unexpected_crash.explanation' defaultMessage='Due to a bug in our code or a browser compatibility issue, this page could not be displayed correctly.' /></p>
           <p><FormattedMessage id='error.unexpected_crash.next_steps' defaultMessage='Try refreshing the page. If that does not help, you may still be able to use Mastodon through a different browser or native app.' /></p>
-          <p className='error-boundary__footer'>Mastodon v{version} · <a href={source_url} rel='noopener' target='_blank'><FormattedMessage id='errors.unexpected_crash.report_issue' defaultMessage='Report issue' /></a> · <button onClick={this.handleCopyStackTrace} className={copied && 'copied'}><FormattedMessage id='errors.unexpected_crash.copy_stacktrace' defaultMessage='Copy stacktrace to clipboard' /></button></p>
+          <p className='error-boundary__footer'>Mastodon v{version} · <a href={source_url} rel='noopener noreferrer' target='_blank'><FormattedMessage id='errors.unexpected_crash.report_issue' defaultMessage='Report issue' /></a> · <button onClick={this.handleCopyStackTrace} className={copied && 'copied'}><FormattedMessage id='errors.unexpected_crash.copy_stacktrace' defaultMessage='Copy stacktrace to clipboard' /></button></p>
         </div>
       </div>
     );
diff --git a/app/javascript/mastodon/components/media_gallery.js b/app/javascript/mastodon/components/media_gallery.js
index e8dd79af9..b8fca8bcb 100644
--- a/app/javascript/mastodon/components/media_gallery.js
+++ b/app/javascript/mastodon/components/media_gallery.js
@@ -159,7 +159,7 @@ class Item extends React.PureComponent {
     if (attachment.get('type') === 'unknown') {
       return (
         <div className={classNames('media-gallery__item', { standalone })} key={attachment.get('id')} style={{ left: left, top: top, right: right, bottom: bottom, width: `${width}%`, height: `${height}%` }}>
-          <a className='media-gallery__item-thumbnail' href={attachment.get('remote_url') || attachment.get('url')} target='_blank' style={{ cursor: 'pointer' }} title={attachment.get('description')}>
+          <a className='media-gallery__item-thumbnail' href={attachment.get('remote_url') || attachment.get('url')} style={{ cursor: 'pointer' }} title={attachment.get('description')} target='_blank' rel='noopener noreferrer'>
             <canvas width={32} height={32} ref={this.setCanvasRef} className='media-gallery__preview' />
           </a>
         </div>
@@ -187,6 +187,7 @@ class Item extends React.PureComponent {
           href={attachment.get('remote_url') || originalUrl}
           onClick={this.handleClick}
           target='_blank'
+          rel='noopener noreferrer'
         >
           <img
             src={previewUrl}
diff --git a/app/javascript/mastodon/components/status.js b/app/javascript/mastodon/components/status.js
index b5606aca5..6cfa96040 100644
--- a/app/javascript/mastodon/components/status.js
+++ b/app/javascript/mastodon/components/status.js
@@ -437,9 +437,9 @@ class Status extends ImmutablePureComponent {
           <div className={classNames('status', `status-${status.get('visibility')}`, { 'status-reply': !!status.get('in_reply_to_id'), muted: this.props.muted, read: unread === false })} data-id={status.get('id')}>
             <div className='status__expand' onClick={this.handleExpandClick} role='presentation' />
             <div className='status__info'>
-              <a href={status.get('url')} className='status__relative-time' target='_blank' rel='noopener'><RelativeTimestamp timestamp={status.get('created_at')} /></a>
+              <a href={status.get('url')} className='status__relative-time' target='_blank' rel='noopener noreferrer'><RelativeTimestamp timestamp={status.get('created_at')} /></a>
 
-              <a onClick={this.handleAccountClick} target='_blank' data-id={status.getIn(['account', 'id'])} href={status.getIn(['account', 'url'])} title={status.getIn(['account', 'acct'])} className='status__display-name'>
+              <a onClick={this.handleAccountClick} data-id={status.getIn(['account', 'id'])} href={status.getIn(['account', 'url'])} title={status.getIn(['account', 'acct'])} className='status__display-name' target='_blank' rel='noopener noreferrer'>
                 <div className='status__avatar'>
                   {statusAvatar}
                 </div>
diff --git a/app/javascript/mastodon/components/status_content.js b/app/javascript/mastodon/components/status_content.js
index 4ce9ec49f..d13091325 100644
--- a/app/javascript/mastodon/components/status_content.js
+++ b/app/javascript/mastodon/components/status_content.js
@@ -59,7 +59,7 @@ export default class StatusContent extends React.PureComponent {
       }
 
       link.setAttribute('target', '_blank');
-      link.setAttribute('rel', 'noopener');
+      link.setAttribute('rel', 'noopener noreferrer');
     }
 
     if (
diff --git a/app/javascript/mastodon/features/account/components/header.js b/app/javascript/mastodon/features/account/components/header.js
index ac97bad71..dbb567e85 100644
--- a/app/javascript/mastodon/features/account/components/header.js
+++ b/app/javascript/mastodon/features/account/components/header.js
@@ -253,7 +253,7 @@ class Header extends ImmutablePureComponent {
 
         <div className='account__header__bar'>
           <div className='account__header__tabs'>
-            <a className='avatar' href={account.get('url')} rel='noopener' target='_blank'>
+            <a className='avatar' href={account.get('url')} rel='noopener noreferrer' target='_blank'>
               <Avatar account={account} size={90} />
             </a>
 
@@ -282,10 +282,10 @@ class Header extends ImmutablePureComponent {
                       <dt dangerouslySetInnerHTML={{ __html: proof.get('provider') }} />
 
                       <dd className='verified'>
-                        <a href={proof.get('proof_url')} target='_blank' rel='noopener'><span title={intl.formatMessage(messages.linkVerifiedOn, { date: intl.formatDate(proof.get('updated_at'), dateFormatOptions) })}>
+                        <a href={proof.get('proof_url')} target='_blank' rel='noopener noreferrer'><span title={intl.formatMessage(messages.linkVerifiedOn, { date: intl.formatDate(proof.get('updated_at'), dateFormatOptions) })}>
                           <Icon id='check' className='verified__mark' />
                         </span></a>
-                        <a href={proof.get('profile_url')} target='_blank' rel='noopener'><span dangerouslySetInnerHTML={{ __html: ' '+proof.get('provider_username') }} /></a>
+                        <a href={proof.get('profile_url')} target='_blank' rel='noopener noreferrer'><span dangerouslySetInnerHTML={{ __html: ' '+proof.get('provider_username') }} /></a>
                       </dd>
                     </dl>
                   ))}
diff --git a/app/javascript/mastodon/features/account_gallery/components/media_item.js b/app/javascript/mastodon/features/account_gallery/components/media_item.js
index b6eec2243..617a45d16 100644
--- a/app/javascript/mastodon/features/account_gallery/components/media_item.js
+++ b/app/javascript/mastodon/features/account_gallery/components/media_item.js
@@ -1,12 +1,12 @@
-import React from 'react';
-import PropTypes from 'prop-types';
-import ImmutablePropTypes from 'react-immutable-proptypes';
-import ImmutablePureComponent from 'react-immutable-pure-component';
+import { decode } from 'blurhash';
+import classNames from 'classnames';
 import Icon from 'mastodon/components/icon';
 import { autoPlayGif, displayMedia } from 'mastodon/initial_state';
-import classNames from 'classnames';
-import { decode } from 'blurhash';
 import { isIOS } from 'mastodon/is_mobile';
+import PropTypes from 'prop-types';
+import React from 'react';
+import ImmutablePropTypes from 'react-immutable-proptypes';
+import ImmutablePureComponent from 'react-immutable-pure-component';
 
 export default class MediaItem extends ImmutablePureComponent {
 
@@ -151,7 +151,7 @@ export default class MediaItem extends ImmutablePureComponent {
 
     return (
       <div className='account-gallery__item' style={{ width, height }}>
-        <a className='media-gallery__item-thumbnail' href={status.get('url')} target='_blank' onClick={this.handleClick} title={title}>
+        <a className='media-gallery__item-thumbnail' href={status.get('url')} onClick={this.handleClick} title={title} target='_blank' rel='noopener noreferrer'>
           <canvas width={32} height={32} ref={this.setCanvasRef} className={classNames('media-gallery__preview', { 'media-gallery__preview--hidden': visible && loaded })} />
           {visible && thumbnail}
           {!visible && icon}
diff --git a/app/javascript/mastodon/features/status/components/card.js b/app/javascript/mastodon/features/status/components/card.js
index 0eff54411..2993fe29a 100644
--- a/app/javascript/mastodon/features/status/components/card.js
+++ b/app/javascript/mastodon/features/status/components/card.js
@@ -148,7 +148,7 @@ export default class Card extends React.PureComponent {
     const horizontal  = (!compact && card.get('width') > card.get('height') && (card.get('width') + 100 >= width)) || card.get('type') !== 'link' || embedded;
     const interactive = card.get('type') !== 'link';
     const className   = classnames('status-card', { horizontal, compact, interactive });
-    const title       = interactive ? <a className='status-card__title' href={card.get('url')} title={card.get('title')} rel='noopener' target='_blank'><strong>{card.get('title')}</strong></a> : <strong className='status-card__title' title={card.get('title')}>{card.get('title')}</strong>;
+    const title       = interactive ? <a className='status-card__title' href={card.get('url')} title={card.get('title')} rel='noopener noreferrer' target='_blank'><strong>{card.get('title')}</strong></a> : <strong className='status-card__title' title={card.get('title')}>{card.get('title')}</strong>;
     const ratio       = card.get('width') / card.get('height');
     const height      = (compact && !embedded) ? (width / (16 / 9)) : (width / ratio);
 
@@ -180,7 +180,7 @@ export default class Card extends React.PureComponent {
             <div className='status-card__actions'>
               <div>
                 <button onClick={this.handleEmbedClick}><Icon id={iconVariant} /></button>
-                {horizontal && <a href={card.get('url')} target='_blank' rel='noopener'><Icon id='external-link' /></a>}
+                {horizontal && <a href={card.get('url')} target='_blank' rel='noopener noreferrer'><Icon id='external-link' /></a>}
               </div>
             </div>
           </div>
@@ -208,7 +208,7 @@ export default class Card extends React.PureComponent {
     }
 
     return (
-      <a href={card.get('url')} className={className} target='_blank' rel='noopener' ref={this.setRef}>
+      <a href={card.get('url')} className={className} target='_blank' rel='noopener noreferrer' ref={this.setRef}>
         {embed}
         {description}
       </a>
diff --git a/app/javascript/mastodon/features/status/components/detailed_status.js b/app/javascript/mastodon/features/status/components/detailed_status.js
index e97f18f08..d5bc82735 100644
--- a/app/javascript/mastodon/features/status/components/detailed_status.js
+++ b/app/javascript/mastodon/features/status/components/detailed_status.js
@@ -156,7 +156,7 @@ export default class DetailedStatus extends ImmutablePureComponent {
     }
 
     if (status.get('application')) {
-      applicationLink = <span> · <a className='detailed-status__application' href={status.getIn(['application', 'website'])} target='_blank' rel='noopener'>{status.getIn(['application', 'name'])}</a></span>;
+      applicationLink = <span> · <a className='detailed-status__application' href={status.getIn(['application', 'website'])} target='_blank' rel='noopener noreferrer'>{status.getIn(['application', 'name'])}</a></span>;
     }
 
     if (status.get('visibility') === 'direct') {
@@ -220,7 +220,7 @@ export default class DetailedStatus extends ImmutablePureComponent {
           {media}
 
           <div className='detailed-status__meta'>
-            <a className='detailed-status__datetime' href={status.get('url')} target='_blank' rel='noopener'>
+            <a className='detailed-status__datetime' href={status.get('url')} target='_blank' rel='noopener noreferrer'>
               <FormattedDate value={new Date(status.get('created_at'))} hour12={false} year='numeric' month='short' day='2-digit' hour='2-digit' minute='2-digit' />
             </a>{applicationLink} · {reblogLink} · {favouriteLink}
           </div>
diff --git a/app/javascript/mastodon/features/ui/components/actions_modal.js b/app/javascript/mastodon/features/ui/components/actions_modal.js
index 00280f7a6..875b2b75d 100644
--- a/app/javascript/mastodon/features/ui/components/actions_modal.js
+++ b/app/javascript/mastodon/features/ui/components/actions_modal.js
@@ -26,7 +26,7 @@ export default class ActionsModal extends ImmutablePureComponent {
 
     return (
       <li key={`${text}-${i}`}>
-        <a href={href} target='_blank' rel='noopener' onClick={this.props.onClick} data-index={i} className={classNames({ active })}>
+        <a href={href} target='_blank' rel='noopener noreferrer' onClick={this.props.onClick} data-index={i} className={classNames({ active })}>
           {icon && <IconButton title={text} icon={icon} role='presentation' tabIndex='-1' inverted />}
           <div>
             <div className={classNames({ 'actions-modal__item-label': !!meta })}>{text}</div>
@@ -42,7 +42,7 @@ export default class ActionsModal extends ImmutablePureComponent {
       <div className='status light'>
         <div className='boost-modal__status-header'>
           <div className='boost-modal__status-time'>
-            <a href={this.props.status.get('url')} className='status__relative-time' target='_blank' rel='noopener'>
+            <a href={this.props.status.get('url')} className='status__relative-time' target='_blank' rel='noopener noreferrer'>
               <RelativeTimestamp timestamp={this.props.status.get('created_at')} />
             </a>
           </div>
diff --git a/app/javascript/mastodon/features/ui/components/boost_modal.js b/app/javascript/mastodon/features/ui/components/boost_modal.js
index 70f4a1282..0e79005f0 100644
--- a/app/javascript/mastodon/features/ui/components/boost_modal.js
+++ b/app/javascript/mastodon/features/ui/components/boost_modal.js
@@ -61,7 +61,7 @@ class BoostModal extends ImmutablePureComponent {
           <div className='status light'>
             <div className='boost-modal__status-header'>
               <div className='boost-modal__status-time'>
-                <a href={status.get('url')} className='status__relative-time' target='_blank' rel='noopener'><RelativeTimestamp timestamp={status.get('created_at')} /></a>
+                <a href={status.get('url')} className='status__relative-time' target='_blank' rel='noopener noreferrer'><RelativeTimestamp timestamp={status.get('created_at')} /></a>
               </div>
 
               <a onClick={this.handleAccountClick} href={status.getIn(['account', 'url'])} className='status__display-name'>
diff --git a/app/javascript/mastodon/features/ui/components/link_footer.js b/app/javascript/mastodon/features/ui/components/link_footer.js
index 2b9bd3875..3c4cff9f9 100644
--- a/app/javascript/mastodon/features/ui/components/link_footer.js
+++ b/app/javascript/mastodon/features/ui/components/link_footer.js
@@ -62,7 +62,7 @@ class LinkFooter extends React.PureComponent {
           <FormattedMessage
             id='getting_started.open_source_notice'
             defaultMessage='Mastodon is open source software. You can contribute or report issues on GitHub at {github}.'
-            values={{ github: <span><a href={source_url} rel='noopener' target='_blank'>{repository}</a> (v{version})</span> }}
+            values={{ github: <span><a href={source_url} rel='noopener noreferrer' target='_blank'>{repository}</a> (v{version})</span> }}
           />
         </p>
       </div>
diff --git a/app/lib/formatter.rb b/app/lib/formatter.rb
index 990b9f63e..6ba327614 100644
--- a/app/lib/formatter.rb
+++ b/app/lib/formatter.rb
@@ -251,7 +251,7 @@ class Formatter
 
   def link_to_url(entity, options = {})
     url        = Addressable::URI.parse(entity[:url])
-    html_attrs = { target: '_blank', rel: 'nofollow noopener' }
+    html_attrs = { target: '_blank', rel: 'nofollow noopener noreferrer' }
 
     html_attrs[:rel] = "me #{html_attrs[:rel]}" if options[:me]
 
diff --git a/app/lib/sanitize_config.rb b/app/lib/sanitize_config.rb
index aba8ce9f6..77045155e 100644
--- a/app/lib/sanitize_config.rb
+++ b/app/lib/sanitize_config.rb
@@ -45,7 +45,7 @@ class Sanitize
 
       add_attributes: {
         'a' => {
-          'rel' => 'nofollow noopener',
+          'rel' => 'nofollow noopener noreferrer',
           'target' => '_blank',
         },
       },
diff --git a/app/views/about/show.html.haml b/app/views/about/show.html.haml
index 80f4cd828..e0ec98ec9 100644
--- a/app/views/about/show.html.haml
+++ b/app/views/about/show.html.haml
@@ -38,7 +38,7 @@
                 %small= t('about.browse_public_posts')
 
         .directory__tag
-          = link_to 'https://joinmastodon.org/apps', target: '_blank', rel: 'noopener' do
+          = link_to 'https://joinmastodon.org/apps', target: '_blank', rel: 'noopener noreferrer' do
             %h4
               = fa_icon 'tablet fw'
               = t('about.get_apps')
diff --git a/app/views/accounts/_moved.html.haml b/app/views/accounts/_moved.html.haml
index 02fd7bf42..a82f277b1 100644
--- a/app/views/accounts/_moved.html.haml
+++ b/app/views/accounts/_moved.html.haml
@@ -6,7 +6,7 @@
     = t('accounts.moved_html', name: content_tag(:bdi, content_tag(:strong, display_name(account, custom_emojify: true), class: :emojify)), new_profile_link: link_to(content_tag(:strong, safe_join(['@', content_tag(:span, moved_to_account.acct)])), ActivityPub::TagManager.instance.url_for(moved_to_account), class: 'mention'))
 
   .moved-account-widget__card
-    = link_to ActivityPub::TagManager.instance.url_for(moved_to_account), class: 'detailed-status__display-name p-author h-card', target: '_blank', rel: 'me noopener' do
+    = link_to ActivityPub::TagManager.instance.url_for(moved_to_account), class: 'detailed-status__display-name p-author h-card', target: '_blank', rel: 'me noopener noreferrer' do
       .detailed-status__display-avatar
         .account__avatar-overlay
           .account__avatar-overlay-base{ style: "background-image: url('#{moved_to_account.avatar.url(:original)}')" }
diff --git a/app/views/admin/reports/_status.html.haml b/app/views/admin/reports/_status.html.haml
index 6facc0a56..425d315e1 100644
--- a/app/views/admin/reports/_status.html.haml
+++ b/app/views/admin/reports/_status.html.haml
@@ -19,7 +19,7 @@
         = react_component :media_gallery, height: 343, sensitive: !current_account&.user&.show_all_media? && status.proper.sensitive? || current_account&.user&.hide_all_media?, 'autoPlayGif': current_account&.user&.setting_auto_play_gif, media: status.proper.media_attachments.map { |a| ActiveModelSerializers::SerializableResource.new(a, serializer: REST::MediaAttachmentSerializer).as_json }
 
     .detailed-status__meta
-      = link_to ActivityPub::TagManager.instance.url_for(status), class: 'detailed-status__datetime', target: stream_link_target, rel: 'noopener' do
+      = link_to ActivityPub::TagManager.instance.url_for(status), class: 'detailed-status__datetime', target: stream_link_target, rel: 'noopener noreferrer' do
         %time.formatted{ datetime: status.created_at.iso8601, title: l(status.created_at) }= l(status.created_at)
       - if status.discarded?
         ·
diff --git a/app/views/admin/tags/show.html.haml b/app/views/admin/tags/show.html.haml
index 5799e5973..c9a147587 100644
--- a/app/views/admin/tags/show.html.haml
+++ b/app/views/admin/tags/show.html.haml
@@ -3,7 +3,7 @@
 
 .dashboard__counters
   %div
-    = link_to tag_url(@tag), target: '_blank', rel: 'noopener' do
+    = link_to tag_url(@tag), target: '_blank', rel: 'noopener noreferrer' do
       .dashboard__counters__num= number_with_delimiter @accounts_today
       .dashboard__counters__label= t 'admin.tags.accounts_today'
   %div
diff --git a/app/views/application/_card.html.haml b/app/views/application/_card.html.haml
index 8719ce484..808dce514 100644
--- a/app/views/application/_card.html.haml
+++ b/app/views/application/_card.html.haml
@@ -1,7 +1,7 @@
 - account_url = local_assigns[:admin] ? admin_account_path(account.id) : ActivityPub::TagManager.instance.url_for(account)
 
 .card.h-card
-  = link_to account_url, target: '_blank', rel: 'noopener' do
+  = link_to account_url, target: '_blank', rel: 'noopener noreferrer' do
     .card__img
       = image_tag account.header.url, alt: ''
     .card__bar
diff --git a/app/views/oauth/authorized_applications/index.html.haml b/app/views/oauth/authorized_applications/index.html.haml
index 7203d758d..7b77108a9 100644
--- a/app/views/oauth/authorized_applications/index.html.haml
+++ b/app/views/oauth/authorized_applications/index.html.haml
@@ -16,7 +16,7 @@
             - if application.website.blank?
               = application.name
             - else
-              = link_to application.name, application.website, target: '_blank', rel: 'noopener'
+              = link_to application.name, application.website, target: '_blank', rel: 'noopener noreferrer'
           %th!= application.scopes.map { |scope| t(scope, scope: [:doorkeeper, :scopes]) }.join(', ')
           %td= l application.created_at
           %td
diff --git a/app/views/statuses/_detailed_status.html.haml b/app/views/statuses/_detailed_status.html.haml
index 5cee84ada..3fa52d3f2 100644
--- a/app/views/statuses/_detailed_status.html.haml
+++ b/app/views/statuses/_detailed_status.html.haml
@@ -44,14 +44,14 @@
   .detailed-status__meta
     %data.dt-published{ value: status.created_at.to_time.iso8601 }
 
-    = link_to ActivityPub::TagManager.instance.url_for(status), class: 'detailed-status__datetime u-url u-uid', target: stream_link_target, rel: 'noopener' do
+    = link_to ActivityPub::TagManager.instance.url_for(status), class: 'detailed-status__datetime u-url u-uid', target: stream_link_target, rel: 'noopener noreferrer' do
       %time.formatted{ datetime: status.created_at.iso8601, title: l(status.created_at) }= l(status.created_at)
     ·
     - if status.application && @account.user&.setting_show_application
       - if status.application.website.blank?
         %strong.detailed-status__application= status.application.name
       - else
-        = link_to status.application.name, status.application.website, class: 'detailed-status__application', target: '_blank', rel: 'noopener'
+        = link_to status.application.name, status.application.website, class: 'detailed-status__application', target: '_blank', rel: 'noopener noreferrer'
       ·
     = link_to remote_interaction_path(status, type: :reply), class: 'modal-button detailed-status__link' do
       - if status.in_reply_to_id.nil?
diff --git a/app/views/statuses/_simple_status.html.haml b/app/views/statuses/_simple_status.html.haml
index a68fe1022..edcfbba2b 100644
--- a/app/views/statuses/_simple_status.html.haml
+++ b/app/views/statuses/_simple_status.html.haml
@@ -1,11 +1,11 @@
 .status
   .status__info
-    = link_to ActivityPub::TagManager.instance.url_for(status), class: 'status__relative-time u-url u-uid', target: stream_link_target, rel: 'noopener' do
+    = link_to ActivityPub::TagManager.instance.url_for(status), class: 'status__relative-time u-url u-uid', target: stream_link_target, rel: 'noopener noreferrer' do
       %time.time-ago{ datetime: status.created_at.iso8601, title: l(status.created_at) }= l(status.created_at)
     %data.dt-published{ value: status.created_at.to_time.iso8601 }
 
     .p-author.h-card
-      = link_to ActivityPub::TagManager.instance.url_for(status.account), class: 'status__display-name u-url', target: stream_link_target, rel: 'noopener' do
+      = link_to ActivityPub::TagManager.instance.url_for(status.account), class: 'status__display-name u-url', target: stream_link_target, rel: 'noopener noreferrer' do
         .status__avatar
           %div
             - if current_account&.user&.setting_auto_play_gif || autoplay
diff --git a/spec/fixtures/xml/mastodon.atom b/spec/fixtures/xml/mastodon.atom
index 9ece3bc2e..92921a938 100644
--- a/spec/fixtures/xml/mastodon.atom
+++ b/spec/fixtures/xml/mastodon.atom
@@ -123,7 +123,7 @@
     <published>2016-10-10T00:41:31Z</published>
     <updated>2016-10-10T00:41:31Z</updated>
     <title>Social media needs MOAR cats! http://kickass.zone/media/3</title>
-    <content type="html">&lt;p&gt;Social media needs MOAR cats! &lt;a rel="nofollow noopener" href="http://kickass.zone/media/3"&gt;http://kickass.zone/media/3&lt;/a&gt;&lt;/p&gt;</content>
+    <content type="html">&lt;p&gt;Social media needs MOAR cats! &lt;a rel="nofollow noopener noreferrer" href="http://kickass.zone/media/3"&gt;http://kickass.zone/media/3&lt;/a&gt;&lt;/p&gt;</content>
     <activity:verb>http://activitystrea.ms/schema/1.0/post</activity:verb>
     <link rel="self" type="application/atom+xml" href="http://kickass.zone/users/localhost/updates/9.atom"/>
     <link rel="alternate" type="text/html" href="http://kickass.zone/users/localhost/updates/9"/>
@@ -135,7 +135,7 @@
     <published>2016-10-10T00:38:39Z</published>
     <updated>2016-10-10T00:38:39Z</updated>
     <title>http://kickass.zone/media/2</title>
-    <content type="html">&lt;p&gt;&lt;a rel="nofollow noopener" href="http://kickass.zone/media/2"&gt;http://kickass.zone/media/2&lt;/a&gt;&lt;/p&gt;</content>
+    <content type="html">&lt;p&gt;&lt;a rel="nofollow noopener noreferrer" href="http://kickass.zone/media/2"&gt;http://kickass.zone/media/2&lt;/a&gt;&lt;/p&gt;</content>
     <activity:verb>http://activitystrea.ms/schema/1.0/post</activity:verb>
     <link rel="self" type="application/atom+xml" href="http://kickass.zone/users/localhost/updates/8.atom"/>
     <link rel="alternate" type="text/html" href="http://kickass.zone/users/localhost/updates/8"/>
diff --git a/spec/lib/sanitize_config_spec.rb b/spec/lib/sanitize_config_spec.rb
index 54bd8693c..feb86af35 100644
--- a/spec/lib/sanitize_config_spec.rb
+++ b/spec/lib/sanitize_config_spec.rb
@@ -24,7 +24,7 @@ describe Sanitize::Config do
     end
 
     it 'keep links in lists' do
-      expect(Sanitize.fragment('<p>Check out:</p><ul><li><a href="https://joinmastodon.org" rel="nofollow noopener" target="_blank">joinmastodon.org</a></li><li>Bar</li></ul>', subject)).to eq '<p>Check out:</p><p><a href="https://joinmastodon.org" rel="nofollow noopener" target="_blank">joinmastodon.org</a><br>Bar</p>'
+      expect(Sanitize.fragment('<p>Check out:</p><ul><li><a href="https://joinmastodon.org" rel="nofollow noopener noreferrer" target="_blank">joinmastodon.org</a></li><li>Bar</li></ul>', subject)).to eq '<p>Check out:</p><p><a href="https://joinmastodon.org" rel="nofollow noopener noreferrer" target="_blank">joinmastodon.org</a><br>Bar</p>'
     end
   end
 end
diff --git a/spec/services/fetch_link_card_service_spec.rb b/spec/services/fetch_link_card_service_spec.rb
index 9761c5f06..3c8f6f578 100644
--- a/spec/services/fetch_link_card_service_spec.rb
+++ b/spec/services/fetch_link_card_service_spec.rb
@@ -80,7 +80,7 @@ RSpec.describe FetchLinkCardService, type: :service do
   end
 
   context 'in a remote status' do
-    let(:status) { Fabricate(:status, account: Fabricate(:account, domain: 'example.com'), text: 'Habt ihr ein paar gute Links zu #<span class="tag"><a href="https://quitter.se/tag/wannacry" target="_blank" rel="tag noopener" title="https://quitter.se/tag/wannacry">Wannacry</a></span> herumfliegen?   Ich will mal unter <br> <a href="https://github.com/qbi/WannaCry" target="_blank" rel="noopener" title="https://github.com/qbi/WannaCry">https://github.com/qbi/WannaCry</a> was sammeln. !<a href="http://sn.jonkman.ca/group/416/id" target="_blank" rel="noopener" title="http://sn.jonkman.ca/group/416/id">security</a>&nbsp;') }
+    let(:status) { Fabricate(:status, account: Fabricate(:account, domain: 'example.com'), text: 'Habt ihr ein paar gute Links zu #<span class="tag"><a href="https://quitter.se/tag/wannacry" target="_blank" rel="tag noopener noreferrer" title="https://quitter.se/tag/wannacry">Wannacry</a></span> herumfliegen?   Ich will mal unter <br> <a href="https://github.com/qbi/WannaCry" target="_blank" rel="noopener noreferrer" title="https://github.com/qbi/WannaCry">https://github.com/qbi/WannaCry</a> was sammeln. !<a href="http://sn.jonkman.ca/group/416/id" target="_blank" rel="noopener noreferrer" title="http://sn.jonkman.ca/group/416/id">security</a>&nbsp;') }
 
     it 'parses out URLs' do
       expect(a_request(:get, 'https://github.com/qbi/WannaCry')).to have_been_made.at_least_once
diff --git a/spec/services/verify_link_service_spec.rb b/spec/services/verify_link_service_spec.rb
index 2edcdb75f..3fc88e60e 100644
--- a/spec/services/verify_link_service_spec.rb
+++ b/spec/services/verify_link_service_spec.rb
@@ -28,12 +28,12 @@ RSpec.describe VerifyLinkService, type: :service do
       end
     end
 
-    context 'when a link contains an <a rel="noopener"> back' do
+    context 'when a link contains an <a rel="noopener noreferrer"> back' do
       let(:html) do
         <<-HTML
           <!doctype html>
           <body>
-            <a href="#{ActivityPub::TagManager.instance.url_for(account)}" rel="noopener me" target="_blank">Follow me on Mastodon</a>
+            <a href="#{ActivityPub::TagManager.instance.url_for(account)}" rel="me noopener noreferrer" target="_blank">Follow me on Mastodon</a>
           </body>
         HTML
       end
-- 
cgit 


From a6269b2f83e3eed1a8ab545f5756cd7b582075f5 Mon Sep 17 00:00:00 2001
From: Takeshi Umeda <noel.yoshiba@gmail.com>
Date: Fri, 25 Oct 2019 05:50:09 +0900
Subject: Split AccountsHelper from StatusesHelper (#12078)

---
 app/helpers/accounts_helper.rb                     | 106 +++++++++++++++++++++
 app/helpers/statuses_helper.rb                     | 103 --------------------
 app/mailers/admin_mailer.rb                        |   2 +-
 app/mailers/notification_mailer.rb                 |   1 +
 app/mailers/user_mailer.rb                         |   2 +-
 app/serializers/rss/account_serializer.rb          |   2 +-
 app/serializers/rss/tag_serializer.rb              |   1 -
 spec/helpers/accounts_helper_spec.rb               |  67 +++++++++++++
 .../admin/account_moderation_notes_helper_spec.rb  |   2 +-
 spec/helpers/statuses_helper_spec.rb               |  54 -----------
 10 files changed, 178 insertions(+), 162 deletions(-)
 create mode 100644 app/helpers/accounts_helper.rb
 create mode 100644 spec/helpers/accounts_helper_spec.rb

(limited to 'spec')

diff --git a/app/helpers/accounts_helper.rb b/app/helpers/accounts_helper.rb
new file mode 100644
index 000000000..5b060a181
--- /dev/null
+++ b/app/helpers/accounts_helper.rb
@@ -0,0 +1,106 @@
+# frozen_string_literal: true
+
+module AccountsHelper
+  def display_name(account, **options)
+    if options[:custom_emojify]
+      Formatter.instance.format_display_name(account, options)
+    else
+      account.display_name.presence || account.username
+    end
+  end
+
+  def acct(account)
+    if account.local?
+      "@#{account.acct}@#{Rails.configuration.x.local_domain}"
+    else
+      "@#{account.acct}"
+    end
+  end
+
+  def account_action_button(account)
+    if user_signed_in?
+      if account.id == current_user.account_id
+        link_to settings_profile_url, class: 'button logo-button' do
+          safe_join([svg_logo, t('settings.edit_profile')])
+        end
+      elsif current_account.following?(account) || current_account.requested?(account)
+        link_to account_unfollow_path(account), class: 'button logo-button button--destructive', data: { method: :post } do
+          safe_join([svg_logo, t('accounts.unfollow')])
+        end
+      elsif !(account.memorial? || account.moved?)
+        link_to account_follow_path(account), class: "button logo-button#{account.blocking?(current_account) ? ' disabled' : ''}", data: { method: :post } do
+          safe_join([svg_logo, t('accounts.follow')])
+        end
+      end
+    elsif !(account.memorial? || account.moved?)
+      link_to account_remote_follow_path(account), class: 'button logo-button modal-button', target: '_new' do
+        safe_join([svg_logo, t('accounts.follow')])
+      end
+    end
+  end
+
+  def minimal_account_action_button(account)
+    if user_signed_in?
+      return if account.id == current_user.account_id
+
+      if current_account.following?(account) || current_account.requested?(account)
+        link_to account_unfollow_path(account), class: 'icon-button active', data: { method: :post }, title: t('accounts.unfollow') do
+          fa_icon('user-times fw')
+        end
+      elsif !(account.memorial? || account.moved?)
+        link_to account_follow_path(account), class: "icon-button#{account.blocking?(current_account) ? ' disabled' : ''}", data: { method: :post }, title: t('accounts.follow') do
+          fa_icon('user-plus fw')
+        end
+      end
+    elsif !(account.memorial? || account.moved?)
+      link_to account_remote_follow_path(account), class: 'icon-button modal-button', target: '_new', title: t('accounts.follow') do
+        fa_icon('user-plus fw')
+      end
+    end
+  end
+
+  def account_badge(account, all: false)
+    if account.bot?
+      content_tag(:div, content_tag(:div, t('accounts.roles.bot'), class: 'account-role bot'), class: 'roles')
+    elsif (Setting.show_staff_badge && account.user_staff?) || all
+      content_tag(:div, class: 'roles') do
+        if all && !account.user_staff?
+          content_tag(:div, t('admin.accounts.roles.user'), class: 'account-role')
+        elsif account.user_admin?
+          content_tag(:div, t('accounts.roles.admin'), class: 'account-role admin')
+        elsif account.user_moderator?
+          content_tag(:div, t('accounts.roles.moderator'), class: 'account-role moderator')
+        end
+      end
+    end
+  end
+
+  def account_description(account)
+    prepend_str = [
+      [
+        number_to_human(account.statuses_count, strip_insignificant_zeros: true),
+        I18n.t('accounts.posts', count: account.statuses_count),
+      ].join(' '),
+
+      [
+        number_to_human(account.following_count, strip_insignificant_zeros: true),
+        I18n.t('accounts.following', count: account.following_count),
+      ].join(' '),
+
+      [
+        number_to_human(account.followers_count, strip_insignificant_zeros: true),
+        I18n.t('accounts.followers', count: account.followers_count),
+      ].join(' '),
+    ].join(', ')
+
+    [prepend_str, account.note].join(' · ')
+  end
+
+  def svg_logo
+    content_tag(:svg, tag(:use, 'xlink:href' => '#mastodon-svg-logo'), 'viewBox' => '0 0 216.4144 232.00976')
+  end
+
+  def svg_logo_full
+    content_tag(:svg, tag(:use, 'xlink:href' => '#mastodon-svg-logo-full'), 'viewBox' => '0 0 713.35878 175.8678')
+  end
+end
diff --git a/app/helpers/statuses_helper.rb b/app/helpers/statuses_helper.rb
index 8380b3c42..866a9902c 100644
--- a/app/helpers/statuses_helper.rb
+++ b/app/helpers/statuses_helper.rb
@@ -4,80 +4,6 @@ module StatusesHelper
   EMBEDDED_CONTROLLER = 'statuses'
   EMBEDDED_ACTION = 'embed'
 
-  def display_name(account, **options)
-    if options[:custom_emojify]
-      Formatter.instance.format_display_name(account, options)
-    else
-      account.display_name.presence || account.username
-    end
-  end
-
-  def account_action_button(account)
-    if user_signed_in?
-      if account.id == current_user.account_id
-        link_to settings_profile_url, class: 'button logo-button' do
-          safe_join([svg_logo, t('settings.edit_profile')])
-        end
-      elsif current_account.following?(account) || current_account.requested?(account)
-        link_to account_unfollow_path(account), class: 'button logo-button button--destructive', data: { method: :post } do
-          safe_join([svg_logo, t('accounts.unfollow')])
-        end
-      elsif !(account.memorial? || account.moved?)
-        link_to account_follow_path(account), class: "button logo-button#{account.blocking?(current_account) ? ' disabled' : ''}", data: { method: :post } do
-          safe_join([svg_logo, t('accounts.follow')])
-        end
-      end
-    elsif !(account.memorial? || account.moved?)
-      link_to account_remote_follow_path(account), class: 'button logo-button modal-button', target: '_new' do
-        safe_join([svg_logo, t('accounts.follow')])
-      end
-    end
-  end
-
-  def minimal_account_action_button(account)
-    if user_signed_in?
-      return if account.id == current_user.account_id
-
-      if current_account.following?(account) || current_account.requested?(account)
-        link_to account_unfollow_path(account), class: 'icon-button active', data: { method: :post }, title: t('accounts.unfollow') do
-          fa_icon('user-times fw')
-        end
-      elsif !(account.memorial? || account.moved?)
-        link_to account_follow_path(account), class: "icon-button#{account.blocking?(current_account) ? ' disabled' : ''}", data: { method: :post }, title: t('accounts.follow') do
-          fa_icon('user-plus fw')
-        end
-      end
-    elsif !(account.memorial? || account.moved?)
-      link_to account_remote_follow_path(account), class: 'icon-button modal-button', target: '_new', title: t('accounts.follow') do
-        fa_icon('user-plus fw')
-      end
-    end
-  end
-
-  def svg_logo
-    content_tag(:svg, tag(:use, 'xlink:href' => '#mastodon-svg-logo'), 'viewBox' => '0 0 216.4144 232.00976')
-  end
-
-  def svg_logo_full
-    content_tag(:svg, tag(:use, 'xlink:href' => '#mastodon-svg-logo-full'), 'viewBox' => '0 0 713.35878 175.8678')
-  end
-
-  def account_badge(account, all: false)
-    if account.bot?
-      content_tag(:div, content_tag(:div, t('accounts.roles.bot'), class: 'account-role bot'), class: 'roles')
-    elsif (Setting.show_staff_badge && account.user_staff?) || all
-      content_tag(:div, class: 'roles') do
-        if all && !account.user_staff?
-          content_tag(:div, t('admin.accounts.roles.user'), class: 'account-role')
-        elsif account.user_admin?
-          content_tag(:div, t('accounts.roles.admin'), class: 'account-role admin')
-        elsif account.user_moderator?
-          content_tag(:div, t('accounts.roles.moderator'), class: 'account-role moderator')
-        end
-      end
-    end
-  end
-
   def link_to_more(url)
     link_to t('statuses.show_more'), url, class: 'load-more load-gap'
   end
@@ -88,27 +14,6 @@ module StatusesHelper
     end
   end
 
-  def account_description(account)
-    prepend_str = [
-      [
-        number_to_human(account.statuses_count, strip_insignificant_zeros: true),
-        I18n.t('accounts.posts', count: account.statuses_count),
-      ].join(' '),
-
-      [
-        number_to_human(account.following_count, strip_insignificant_zeros: true),
-        I18n.t('accounts.following', count: account.following_count),
-      ].join(' '),
-
-      [
-        number_to_human(account.followers_count, strip_insignificant_zeros: true),
-        I18n.t('accounts.followers', count: account.followers_count),
-      ].join(' '),
-    ].join(', ')
-
-    [prepend_str, account.note].join(' · ')
-  end
-
   def media_summary(status)
     attachments = { image: 0, video: 0 }
 
@@ -154,14 +59,6 @@ module StatusesHelper
     embedded_view? ? '_blank' : nil
   end
 
-  def acct(account)
-    if account.local?
-      "@#{account.acct}@#{Rails.configuration.x.local_domain}"
-    else
-      "@#{account.acct}"
-    end
-  end
-
   def style_classes(status, is_predecessor, is_successor, include_threads)
     classes = ['entry']
     classes << 'entry-predecessor' if is_predecessor
diff --git a/app/mailers/admin_mailer.rb b/app/mailers/admin_mailer.rb
index 8abce5f05..11fd09e30 100644
--- a/app/mailers/admin_mailer.rb
+++ b/app/mailers/admin_mailer.rb
@@ -3,7 +3,7 @@
 class AdminMailer < ApplicationMailer
   layout 'plain_mailer'
 
-  helper :statuses
+  helper :accounts
 
   def new_report(recipient, report)
     @report   = report
diff --git a/app/mailers/notification_mailer.rb b/app/mailers/notification_mailer.rb
index 723d901fc..9d8a7886c 100644
--- a/app/mailers/notification_mailer.rb
+++ b/app/mailers/notification_mailer.rb
@@ -1,6 +1,7 @@
 # frozen_string_literal: true
 
 class NotificationMailer < ApplicationMailer
+  helper :accounts
   helper :statuses
 
   add_template_helper RoutingHelper
diff --git a/app/mailers/user_mailer.rb b/app/mailers/user_mailer.rb
index 6b81f6873..c30bec80b 100644
--- a/app/mailers/user_mailer.rb
+++ b/app/mailers/user_mailer.rb
@@ -3,9 +3,9 @@
 class UserMailer < Devise::Mailer
   layout 'mailer'
 
+  helper :accounts
   helper :application
   helper :instance
-  helper :statuses
 
   add_template_helper RoutingHelper
 
diff --git a/app/serializers/rss/account_serializer.rb b/app/serializers/rss/account_serializer.rb
index e39b2b372..ee972ff96 100644
--- a/app/serializers/rss/account_serializer.rb
+++ b/app/serializers/rss/account_serializer.rb
@@ -2,7 +2,7 @@
 
 class RSS::AccountSerializer
   include ActionView::Helpers::NumberHelper
-  include StatusesHelper
+  include AccountsHelper
   include RoutingHelper
 
   def render(account, statuses, tag)
diff --git a/app/serializers/rss/tag_serializer.rb b/app/serializers/rss/tag_serializer.rb
index 6737fb2c9..ea26189a2 100644
--- a/app/serializers/rss/tag_serializer.rb
+++ b/app/serializers/rss/tag_serializer.rb
@@ -3,7 +3,6 @@
 class RSS::TagSerializer
   include ActionView::Helpers::NumberHelper
   include ActionView::Helpers::SanitizeHelper
-  include StatusesHelper
   include RoutingHelper
 
   def render(tag, statuses)
diff --git a/spec/helpers/accounts_helper_spec.rb b/spec/helpers/accounts_helper_spec.rb
new file mode 100644
index 000000000..2b35b23b7
--- /dev/null
+++ b/spec/helpers/accounts_helper_spec.rb
@@ -0,0 +1,67 @@
+require 'rails_helper'
+
+RSpec.describe AccountsHelper, type: :helper do
+  def set_not_embedded_view
+    params[:controller] = "not_#{StatusesHelper::EMBEDDED_CONTROLLER}"
+    params[:action] = "not_#{StatusesHelper::EMBEDDED_ACTION}"
+  end
+
+  def set_embedded_view
+    params[:controller] = StatusesHelper::EMBEDDED_CONTROLLER
+    params[:action] = StatusesHelper::EMBEDDED_ACTION
+  end
+
+  describe '#display_name' do
+    it 'uses the display name when it exists' do
+      account = Account.new(display_name: "Display", username: "Username")
+
+      expect(helper.display_name(account)).to eq "Display"
+    end
+
+    it 'uses the username when display name is nil' do
+      account = Account.new(display_name: nil, username: "Username")
+
+      expect(helper.display_name(account)).to eq "Username"
+    end
+  end
+
+  describe '#acct' do
+    it 'is fully qualified for embedded local accounts' do
+      allow(Rails.configuration.x).to receive(:local_domain).and_return('local_domain')
+      set_embedded_view
+      account = Account.new(domain: nil, username: 'user')
+
+      acct = helper.acct(account)
+
+      expect(acct).to eq '@user@local_domain'
+    end
+
+    it 'is fully qualified for embedded foreign accounts' do
+      set_embedded_view
+      account = Account.new(domain: 'foreign_server.com', username: 'user')
+
+      acct = helper.acct(account)
+
+      expect(acct).to eq '@user@foreign_server.com'
+    end
+
+    it 'is fully qualified for non embedded foreign accounts' do
+      set_not_embedded_view
+      account = Account.new(domain: 'foreign_server.com', username: 'user')
+
+      acct = helper.acct(account)
+
+      expect(acct).to eq '@user@foreign_server.com'
+    end
+
+    it 'is fully qualified for non embedded local accounts' do
+      allow(Rails.configuration.x).to receive(:local_domain).and_return('local_domain')
+      set_not_embedded_view
+      account = Account.new(domain: nil, username: 'user')
+
+      acct = helper.acct(account)
+
+      expect(acct).to eq '@user@local_domain'
+    end
+  end
+end
diff --git a/spec/helpers/admin/account_moderation_notes_helper_spec.rb b/spec/helpers/admin/account_moderation_notes_helper_spec.rb
index ddfe8b46f..622ce8806 100644
--- a/spec/helpers/admin/account_moderation_notes_helper_spec.rb
+++ b/spec/helpers/admin/account_moderation_notes_helper_spec.rb
@@ -3,7 +3,7 @@
 require 'rails_helper'
 
 RSpec.describe Admin::AccountModerationNotesHelper, type: :helper do
-  include StatusesHelper
+  include AccountsHelper
 
   describe '#admin_account_link_to' do
     context 'account is nil' do
diff --git a/spec/helpers/statuses_helper_spec.rb b/spec/helpers/statuses_helper_spec.rb
index 510955a2f..940ff072e 100644
--- a/spec/helpers/statuses_helper_spec.rb
+++ b/spec/helpers/statuses_helper_spec.rb
@@ -1,20 +1,6 @@
 require 'rails_helper'
 
 RSpec.describe StatusesHelper, type: :helper do
-  describe '#display_name' do
-    it 'uses the display name when it exists' do
-      account = Account.new(display_name: "Display", username: "Username")
-
-      expect(helper.display_name(account)).to eq "Display"
-    end
-
-    it 'uses the username when display name is nil' do
-      account = Account.new(display_name: nil, username: "Username")
-
-      expect(helper.display_name(account)).to eq "Username"
-    end
-  end
-
   describe '#stream_link_target' do
     it 'returns nil if it is not an embedded view' do
       set_not_embedded_view
@@ -29,46 +15,6 @@ RSpec.describe StatusesHelper, type: :helper do
     end
   end
 
-  describe '#acct' do
-    it 'is fully qualified for embedded local accounts' do
-      allow(Rails.configuration.x).to receive(:local_domain).and_return('local_domain')
-      set_embedded_view
-      account = Account.new(domain: nil, username: 'user')
-
-      acct = helper.acct(account)
-
-      expect(acct).to eq '@user@local_domain'
-    end
-
-    it 'is fully qualified for embedded foreign accounts' do
-      set_embedded_view
-      account = Account.new(domain: 'foreign_server.com', username: 'user')
-
-      acct = helper.acct(account)
-
-      expect(acct).to eq '@user@foreign_server.com'
-    end
-
-    it 'is fully qualified for non embedded foreign accounts' do
-      set_not_embedded_view
-      account = Account.new(domain: 'foreign_server.com', username: 'user')
-
-      acct = helper.acct(account)
-
-      expect(acct).to eq '@user@foreign_server.com'
-    end
-
-    it 'is fully qualified for non embedded local accounts' do
-      allow(Rails.configuration.x).to receive(:local_domain).and_return('local_domain')
-      set_not_embedded_view
-      account = Account.new(domain: nil, username: 'user')
-
-      acct = helper.acct(account)
-
-      expect(acct).to eq '@user@local_domain'
-    end
-  end
-
   def set_not_embedded_view
     params[:controller] = "not_#{StatusesHelper::EMBEDDED_CONTROLLER}"
     params[:action] = "not_#{StatusesHelper::EMBEDDED_ACTION}"
-- 
cgit 


From 650820d62d46e803083b1178a95ba2a29f7d8b6e Mon Sep 17 00:00:00 2001
From: ThibG <thib@sitedethib.com>
Date: Mon, 4 Nov 2019 13:00:16 +0100
Subject: Fix remote media descriptions being cut off at 420 chars (#12262)

* Fix remote media descriptions being cut off at 420 chars

Fixes #12258

* Fix tests
---
 app/models/media_attachment.rb               |  6 ++++--
 spec/lib/activitypub/activity/create_spec.rb | 26 ++++++++++++++++++++++++++
 spec/models/media_attachment_spec.rb         |  4 ++--
 3 files changed, 32 insertions(+), 4 deletions(-)

(limited to 'spec')

diff --git a/app/models/media_attachment.rb b/app/models/media_attachment.rb
index 137377422..1f9d92b22 100644
--- a/app/models/media_attachment.rb
+++ b/app/models/media_attachment.rb
@@ -26,6 +26,8 @@ class MediaAttachment < ApplicationRecord
 
   enum type: [:image, :gifv, :video, :unknown, :audio]
 
+  MAX_DESCRIPTION_LENGTH = 1_500
+
   IMAGE_FILE_EXTENSIONS = %w(.jpg .jpeg .png .gif).freeze
   VIDEO_FILE_EXTENSIONS = %w(.webm .mp4 .m4v .mov).freeze
   AUDIO_FILE_EXTENSIONS = %w(.ogg .oga .mp3 .wav .flac .opus .aac .m4a .3gp .wma).freeze
@@ -139,7 +141,7 @@ class MediaAttachment < ApplicationRecord
   include Attachmentable
 
   validates :account, presence: true
-  validates :description, length: { maximum: 1_500 }, if: :local?
+  validates :description, length: { maximum: MAX_DESCRIPTION_LENGTH }, if: :local?
 
   scope :attached,   -> { where.not(status_id: nil).or(where.not(scheduled_status_id: nil)) }
   scope :unattached, -> { where(status_id: nil, scheduled_status_id: nil) }
@@ -243,7 +245,7 @@ class MediaAttachment < ApplicationRecord
   end
 
   def prepare_description
-    self.description = description.strip[0...420] unless description.nil?
+    self.description = description.strip[0...MAX_DESCRIPTION_LENGTH] unless description.nil?
   end
 
   def set_type_and_extension
diff --git a/spec/lib/activitypub/activity/create_spec.rb b/spec/lib/activitypub/activity/create_spec.rb
index 412609de4..b709954a3 100644
--- a/spec/lib/activitypub/activity/create_spec.rb
+++ b/spec/lib/activitypub/activity/create_spec.rb
@@ -261,6 +261,32 @@ RSpec.describe ActivityPub::Activity::Create do
         end
       end
 
+
+      context 'with media attachments with long description' do
+        let(:object_json) do
+          {
+            id: [ActivityPub::TagManager.instance.uri_for(sender), '#bar'].join,
+            type: 'Note',
+            content: 'Lorem ipsum',
+            attachment: [
+              {
+                type: 'Document',
+                mediaType: 'image/png',
+                url: 'http://example.com/attachment.png',
+                name: '*' * 1500,
+              },
+            ],
+          }
+        end
+
+        it 'creates status' do
+          status = sender.statuses.first
+
+          expect(status).to_not be_nil
+          expect(status.media_attachments.map(&:description)).to include('*' * 1500)
+        end
+      end
+
       context 'with media attachments with focal points' do
         let(:object_json) do
           {
diff --git a/spec/models/media_attachment_spec.rb b/spec/models/media_attachment_spec.rb
index 266cd4920..7ddfba7ed 100644
--- a/spec/models/media_attachment_spec.rb
+++ b/spec/models/media_attachment_spec.rb
@@ -136,10 +136,10 @@ RSpec.describe MediaAttachment, type: :model do
   end
 
   describe 'descriptions for remote attachments' do
-    it 'are cut off at 140 characters' do
+    it 'are cut off at 1500 characters' do
       media = Fabricate(:media_attachment, description: 'foo' * 1000, remote_url: 'http://example.com/blah.jpg')
 
-      expect(media.description.size).to be <= 420
+      expect(media.description.size).to be <= 1_500
     end
   end
 end
-- 
cgit 


From 66c1fe0495b7e4728ad776e9c41a70c180640ea5 Mon Sep 17 00:00:00 2001
From: ThibG <thib@sitedethib.com>
Date: Thu, 7 Nov 2019 08:05:07 +0100
Subject: Fix various issues with account migration (#12301)

* Fix being able to follow oneself by moving to an account that was following the old one

* Add specs

* Add spec to catch MoveWorker issue with local followers following both accounts

* Fix move worker breaking when a local account follows both source and target accounts

* Fix migration from remote to local account not sending Undo Follow

* Fix show_reblogs not being preserved for moved account's followers
---
 app/services/follow_service.rb              |  4 +-
 app/workers/move_worker.rb                  |  8 +++-
 app/workers/unfollow_follow_worker.rb       |  7 +++-
 spec/workers/move_worker_spec.rb            | 63 +++++++++++++++++++++++++++++
 spec/workers/unfollow_follow_worker_spec.rb | 50 +++++++++++++++++++++++
 5 files changed, 126 insertions(+), 6 deletions(-)
 create mode 100644 spec/workers/move_worker_spec.rb
 create mode 100644 spec/workers/unfollow_follow_worker_spec.rb

(limited to 'spec')

diff --git a/app/services/follow_service.rb b/app/services/follow_service.rb
index 1941c2e2d..dc47804c0 100644
--- a/app/services/follow_service.rb
+++ b/app/services/follow_service.rb
@@ -8,7 +8,7 @@ class FollowService < BaseService
   # @param [Account] source_account From which to follow
   # @param [String, Account] uri User URI to follow in the form of username@domain (or account record)
   # @param [true, false, nil] reblogs Whether or not to show reblogs, defaults to true
-  def call(source_account, target_account, reblogs: nil)
+  def call(source_account, target_account, reblogs: nil, bypass_locked: false)
     reblogs = true if reblogs.nil?
     target_account = ResolveAccountService.new.call(target_account, skip_webfinger: true)
 
@@ -30,7 +30,7 @@ class FollowService < BaseService
 
     ActivityTracker.increment('activity:interactions')
 
-    if target_account.locked? || source_account.silenced? || target_account.activitypub?
+    if (target_account.locked? && !bypass_locked) || source_account.silenced? || target_account.activitypub?
       request_follow(source_account, target_account, reblogs: reblogs)
     elsif target_account.local?
       direct_follow(source_account, target_account, reblogs: reblogs)
diff --git a/app/workers/move_worker.rb b/app/workers/move_worker.rb
index 83f7090ee..595730226 100644
--- a/app/workers/move_worker.rb
+++ b/app/workers/move_worker.rb
@@ -7,7 +7,7 @@ class MoveWorker
     @source_account = Account.find(source_account_id)
     @target_account = Account.find(target_account_id)
 
-    if @target_account.local?
+    if @target_account.local? && @source_account.local?
       rewrite_follows!
     else
       queue_follow_unfollows!
@@ -21,13 +21,17 @@ class MoveWorker
   def rewrite_follows!
     @source_account.passive_relationships
                    .where(account: Account.local)
+                   .where.not(account: @target_account.followers.local)
+                   .where.not(account_id: @target_account.id)
                    .in_batches
                    .update_all(target_account_id: @target_account.id)
   end
 
   def queue_follow_unfollows!
+    bypass_locked = @target_account.local?
+
     @source_account.followers.local.select(:id).find_in_batches do |accounts|
-      UnfollowFollowWorker.push_bulk(accounts.map(&:id)) { |follower_id| [follower_id, @source_account.id, @target_account.id] }
+      UnfollowFollowWorker.push_bulk(accounts.map(&:id)) { |follower_id| [follower_id, @source_account.id, @target_account.id, bypass_locked] }
     end
   end
 end
diff --git a/app/workers/unfollow_follow_worker.rb b/app/workers/unfollow_follow_worker.rb
index 95549e107..b6e665a41 100644
--- a/app/workers/unfollow_follow_worker.rb
+++ b/app/workers/unfollow_follow_worker.rb
@@ -5,12 +5,15 @@ class UnfollowFollowWorker
 
   sidekiq_options queue: 'pull'
 
-  def perform(follower_account_id, old_target_account_id, new_target_account_id)
+  def perform(follower_account_id, old_target_account_id, new_target_account_id, bypass_locked = false)
     follower_account   = Account.find(follower_account_id)
     old_target_account = Account.find(old_target_account_id)
     new_target_account = Account.find(new_target_account_id)
 
-    FollowService.new.call(follower_account, new_target_account)
+    follow = follower_account.active_relationships.find_by(target_account: old_target_account)
+    reblogs = follow&.show_reblogs?
+
+    FollowService.new.call(follower_account, new_target_account, reblogs: reblogs, bypass_locked: bypass_locked)
     UnfollowService.new.call(follower_account, old_target_account, skip_unmerge: true)
   rescue ActiveRecord::RecordNotFound, Mastodon::NotPermittedError
     true
diff --git a/spec/workers/move_worker_spec.rb b/spec/workers/move_worker_spec.rb
new file mode 100644
index 000000000..b8f4d9900
--- /dev/null
+++ b/spec/workers/move_worker_spec.rb
@@ -0,0 +1,63 @@
+# frozen_string_literal: true
+
+require 'rails_helper'
+
+describe MoveWorker do
+  let(:local_follower)   { Fabricate(:user, email: 'bob@example.com', account: Fabricate(:account, username: 'bob')).account }
+  let(:source_account)   { Fabricate(:account, protocol: :activitypub, domain: 'example.com') }
+  let(:target_account)   { Fabricate(:account, protocol: :activitypub, domain: 'example.com') }
+
+  subject { described_class.new }
+
+  before do
+    local_follower.follow!(source_account)
+  end
+
+  context 'both accounts are distant' do
+    describe 'perform' do
+      it 'calls UnfollowFollowWorker' do
+        allow(UnfollowFollowWorker).to receive(:push_bulk)
+        subject.perform(source_account.id, target_account.id)
+        expect(UnfollowFollowWorker).to have_received(:push_bulk).with([local_follower.id])
+      end
+    end
+  end
+
+  context 'target account is local' do
+    let(:target_account) { Fabricate(:user, email: 'alice@example.com', account: Fabricate(:account, username: 'alice')).account }
+
+    describe 'perform' do
+      it 'calls UnfollowFollowWorker' do
+        allow(UnfollowFollowWorker).to receive(:push_bulk)
+        subject.perform(source_account.id, target_account.id)
+        expect(UnfollowFollowWorker).to have_received(:push_bulk).with([local_follower.id])
+      end
+    end
+  end
+
+  context 'both target and source accounts are local' do
+    let(:target_account) { Fabricate(:user, email: 'alice@example.com', account: Fabricate(:account, username: 'alice')).account }
+    let(:source_account) { Fabricate(:user, email: 'alice_@example.com', account: Fabricate(:account, username: 'alice_')).account }
+
+    describe 'perform' do
+      it 'calls makes local followers follow the target account' do
+        subject.perform(source_account.id, target_account.id)
+        expect(local_follower.following?(target_account)).to be true
+      end
+
+      it 'does not fail when a local user is already following both accounts' do
+        double_follower = Fabricate(:user, email: 'eve@example.com', account: Fabricate(:account, username: 'eve')).account
+        double_follower.follow!(source_account)
+        double_follower.follow!(target_account)
+        subject.perform(source_account.id, target_account.id)
+        expect(local_follower.following?(target_account)).to be true
+      end
+
+      it 'does not allow the moved account to follow themselves' do
+        source_account.follow!(target_account)
+        subject.perform(source_account.id, target_account.id)
+        expect(target_account.following?(target_account)).to be false
+      end
+    end
+  end
+end
diff --git a/spec/workers/unfollow_follow_worker_spec.rb b/spec/workers/unfollow_follow_worker_spec.rb
new file mode 100644
index 000000000..5052c5616
--- /dev/null
+++ b/spec/workers/unfollow_follow_worker_spec.rb
@@ -0,0 +1,50 @@
+# frozen_string_literal: true
+
+require 'rails_helper'
+
+describe UnfollowFollowWorker do
+  let(:local_follower)   { Fabricate(:user, email: 'bob@example.com', account: Fabricate(:account, username: 'bob')).account }
+  let(:source_account)   { Fabricate(:account) }
+  let(:target_account)   { Fabricate(:account) }
+  let(:show_reblogs)     { true }
+
+  subject { described_class.new }
+
+  before do
+    local_follower.follow!(source_account, reblogs: show_reblogs)
+  end
+
+  context 'when show_reblogs is true' do
+    let(:show_reblogs) { true }
+
+    describe 'perform' do
+      it 'unfollows source account and follows target account' do
+        subject.perform(local_follower.id, source_account.id, target_account.id)
+        expect(local_follower.following?(source_account)).to be false
+        expect(local_follower.following?(target_account)).to be true
+      end
+
+      it 'preserves show_reblogs' do
+        subject.perform(local_follower.id, source_account.id, target_account.id)
+        expect(Follow.find_by(account: local_follower, target_account: target_account).show_reblogs?).to be show_reblogs
+      end
+    end
+  end
+
+  context 'when show_reblogs is false' do
+    let(:show_reblogs) { false }
+
+    describe 'perform' do
+      it 'unfollows source account and follows target account' do
+        subject.perform(local_follower.id, source_account.id, target_account.id)
+        expect(local_follower.following?(source_account)).to be false
+        expect(local_follower.following?(target_account)).to be true
+      end
+
+      it 'preserves show_reblogs' do
+        subject.perform(local_follower.id, source_account.id, target_account.id)
+        expect(Follow.find_by(account: local_follower, target_account: target_account).show_reblogs?).to be show_reblogs
+      end
+    end
+  end
+end
-- 
cgit 


From afb398b583b23c139c5a069c1281550bb69760e0 Mon Sep 17 00:00:00 2001
From: Yamagishi Kazutoshi <ykzts@desire.sh>
Date: Thu, 14 Nov 2019 06:53:05 +0900
Subject: Change to always returns html document in error pages (#12214)

---
 app/controllers/application_controller.rb       | 5 +----
 spec/controllers/application_controller_spec.rb | 5 -----
 2 files changed, 1 insertion(+), 9 deletions(-)

(limited to 'spec')

diff --git a/app/controllers/application_controller.rb b/app/controllers/application_controller.rb
index bd3d13774..e19d5b142 100644
--- a/app/controllers/application_controller.rb
+++ b/app/controllers/application_controller.rb
@@ -136,9 +136,6 @@ class ApplicationController < ActionController::Base
   end
 
   def respond_with_error(code)
-    respond_to do |format|
-      format.any  { head code }
-      format.html { render "errors/#{code}", layout: 'error', status: code }
-    end
+    render "errors/#{code}", layout: 'error', status: code
   end
 end
diff --git a/spec/controllers/application_controller_spec.rb b/spec/controllers/application_controller_spec.rb
index da4a794cd..63ae27a92 100644
--- a/spec/controllers/application_controller_spec.rb
+++ b/spec/controllers/application_controller_spec.rb
@@ -22,11 +22,6 @@ describe ApplicationController, type: :controller do
   end
 
   shared_examples 'respond_with_error' do |code|
-    it "returns http #{code} for any" do
-      subject
-      expect(response).to have_http_status(code)
-    end
-
     it "returns http #{code} for http" do
       subject
       expect(response).to have_http_status(code)
-- 
cgit 


From dfea7368c934f600bd0b6b93b4a6c008a4e265b0 Mon Sep 17 00:00:00 2001
From: ThibG <thib@sitedethib.com>
Date: Wed, 13 Nov 2019 23:02:10 +0100
Subject: Add bookmarks (#7107)

* Add backend support for bookmarks

Bookmarks behave like favourites, except they aren't shared with other
users and do not have an associated counter.

* Add spec for bookmark endpoints

* Add front-end support for bookmarks

* Introduce OAuth scopes for bookmarks

* Add bookmarks to archive takeout

* Fix migration

* Coding style fixes

* Fix rebase issue

* Update bookmarked_statuses to latest UI changes

* Update bookmark actions to properly reflect status changes in state

* Add bookmarks item to single-column layout

* Make active bookmarks red
---
 app/controllers/api/v1/bookmarks_controller.rb     |  67 +++++++++++++
 .../api/v1/statuses/bookmarks_controller.rb        |  39 ++++++++
 app/javascript/mastodon/actions/bookmarks.js       |  90 ++++++++++++++++++
 app/javascript/mastodon/actions/interactions.js    |  80 ++++++++++++++++
 .../mastodon/components/status_action_bar.js       |   7 ++
 .../mastodon/containers/status_container.js        |  10 ++
 .../mastodon/features/bookmarked_statuses/index.js | 104 +++++++++++++++++++++
 .../mastodon/features/getting_started/index.js     |   4 +-
 .../features/status/components/action_bar.js       |   7 ++
 app/javascript/mastodon/features/status/index.js   |  11 +++
 .../features/ui/components/columns_area.js         |   2 +
 .../features/ui/components/navigation_panel.js     |   1 +
 app/javascript/mastodon/features/ui/index.js       |   2 +
 .../mastodon/features/ui/util/async-components.js  |   4 +
 app/javascript/mastodon/reducers/status_lists.js   |  29 ++++++
 app/javascript/mastodon/reducers/statuses.js       |   6 ++
 app/javascript/styles/mastodon/components.scss     |   4 +
 app/javascript/styles/mastodon/variables.scss      |   2 +
 app/models/bookmark.rb                             |  26 ++++++
 app/models/concerns/account_associations.rb        |   1 +
 app/models/concerns/account_interactions.rb        |   4 +
 app/models/status.rb                               |   5 +
 app/presenters/status_relationships_presenter.rb   |   2 +
 app/serializers/rest/status_serializer.rb          |   9 ++
 app/services/backup_service.rb                     |  21 +++++
 config/initializers/doorkeeper.rb                  |   2 +
 config/locales/doorkeeper.en.yml                   |   2 +
 config/routes.rb                                   |   4 +
 db/migrate/20180831171112_create_bookmarks.rb      |  17 ++++
 db/schema.rb                                       |  12 +++
 .../api/v1/bookmarks_controller_spec.rb            |  78 ++++++++++++++++
 .../api/v1/statuses/bookmarks_controller_spec.rb   |  57 +++++++++++
 spec/fabricators/bookmark_fabricator.rb            |   4 +
 33 files changed, 712 insertions(+), 1 deletion(-)
 create mode 100644 app/controllers/api/v1/bookmarks_controller.rb
 create mode 100644 app/controllers/api/v1/statuses/bookmarks_controller.rb
 create mode 100644 app/javascript/mastodon/actions/bookmarks.js
 create mode 100644 app/javascript/mastodon/features/bookmarked_statuses/index.js
 create mode 100644 app/models/bookmark.rb
 create mode 100644 db/migrate/20180831171112_create_bookmarks.rb
 create mode 100644 spec/controllers/api/v1/bookmarks_controller_spec.rb
 create mode 100644 spec/controllers/api/v1/statuses/bookmarks_controller_spec.rb
 create mode 100644 spec/fabricators/bookmark_fabricator.rb

(limited to 'spec')

diff --git a/app/controllers/api/v1/bookmarks_controller.rb b/app/controllers/api/v1/bookmarks_controller.rb
new file mode 100644
index 000000000..cf4cba8dd
--- /dev/null
+++ b/app/controllers/api/v1/bookmarks_controller.rb
@@ -0,0 +1,67 @@
+# frozen_string_literal: true
+
+class Api::V1::BookmarksController < Api::BaseController
+  before_action -> { doorkeeper_authorize! :read, :'read:bookmarks' }
+  before_action :require_user!
+  after_action :insert_pagination_headers
+
+  respond_to :json
+
+  def index
+    @statuses = load_statuses
+    render json: @statuses, each_serializer: REST::StatusSerializer, relationships: StatusRelationshipsPresenter.new(@statuses, current_user&.account_id)
+  end
+
+  private
+
+  def load_statuses
+    cached_bookmarks
+  end
+
+  def cached_bookmarks
+    cache_collection(
+      Status.reorder(nil).joins(:bookmarks).merge(results),
+      Status
+    )
+  end
+
+  def results
+    @_results ||= account_bookmarks.paginate_by_max_id(
+      limit_param(DEFAULT_STATUSES_LIMIT),
+      params[:max_id],
+      params[:since_id]
+    )
+  end
+
+  def account_bookmarks
+    current_account.bookmarks
+  end
+
+  def insert_pagination_headers
+    set_pagination_headers(next_path, prev_path)
+  end
+
+  def next_path
+    api_v1_bookmarks_url pagination_params(max_id: pagination_max_id) if records_continue?
+  end
+
+  def prev_path
+    api_v1_bookmarks_url pagination_params(since_id: pagination_since_id) unless results.empty?
+  end
+
+  def pagination_max_id
+    results.last.id
+  end
+
+  def pagination_since_id
+    results.first.id
+  end
+
+  def records_continue?
+    results.size == limit_param(DEFAULT_STATUSES_LIMIT)
+  end
+
+  def pagination_params(core_params)
+    params.slice(:limit).permit(:limit).merge(core_params)
+  end
+end
diff --git a/app/controllers/api/v1/statuses/bookmarks_controller.rb b/app/controllers/api/v1/statuses/bookmarks_controller.rb
new file mode 100644
index 000000000..bb9729cf5
--- /dev/null
+++ b/app/controllers/api/v1/statuses/bookmarks_controller.rb
@@ -0,0 +1,39 @@
+# frozen_string_literal: true
+
+class Api::V1::Statuses::BookmarksController < Api::BaseController
+  include Authorization
+
+  before_action -> { doorkeeper_authorize! :write, :'write:bookmarks' }
+  before_action :require_user!
+
+  respond_to :json
+
+  def create
+    @status = bookmarked_status
+    render json: @status, serializer: REST::StatusSerializer
+  end
+
+  def destroy
+    @status = requested_status
+    @bookmarks_map = { @status.id => false }
+
+    bookmark = Bookmark.find_by!(account: current_user.account, status: @status)
+    bookmark.destroy!
+
+    render json: @status, serializer: REST::StatusSerializer, relationships: StatusRelationshipsPresenter.new([@status], current_user&.account_id, bookmarks_map: @bookmarks_map)
+  end
+
+  private
+
+  def bookmarked_status
+    authorize_with current_user.account, requested_status, :show?
+
+    bookmark = Bookmark.find_or_create_by!(account: current_user.account, status: requested_status)
+
+    bookmark.status.reload
+  end
+
+  def requested_status
+    Status.find(params[:status_id])
+  end
+end
diff --git a/app/javascript/mastodon/actions/bookmarks.js b/app/javascript/mastodon/actions/bookmarks.js
new file mode 100644
index 000000000..544ed2ff2
--- /dev/null
+++ b/app/javascript/mastodon/actions/bookmarks.js
@@ -0,0 +1,90 @@
+import api, { getLinks } from '../api';
+import { importFetchedStatuses } from './importer';
+
+export const BOOKMARKED_STATUSES_FETCH_REQUEST = 'BOOKMARKED_STATUSES_FETCH_REQUEST';
+export const BOOKMARKED_STATUSES_FETCH_SUCCESS = 'BOOKMARKED_STATUSES_FETCH_SUCCESS';
+export const BOOKMARKED_STATUSES_FETCH_FAIL    = 'BOOKMARKED_STATUSES_FETCH_FAIL';
+
+export const BOOKMARKED_STATUSES_EXPAND_REQUEST = 'BOOKMARKED_STATUSES_EXPAND_REQUEST';
+export const BOOKMARKED_STATUSES_EXPAND_SUCCESS = 'BOOKMARKED_STATUSES_EXPAND_SUCCESS';
+export const BOOKMARKED_STATUSES_EXPAND_FAIL    = 'BOOKMARKED_STATUSES_EXPAND_FAIL';
+
+export function fetchBookmarkedStatuses() {
+  return (dispatch, getState) => {
+    if (getState().getIn(['status_lists', 'bookmarks', 'isLoading'])) {
+      return;
+    }
+
+    dispatch(fetchBookmarkedStatusesRequest());
+
+    api(getState).get('/api/v1/bookmarks').then(response => {
+      const next = getLinks(response).refs.find(link => link.rel === 'next');
+      dispatch(importFetchedStatuses(response.data));
+      dispatch(fetchBookmarkedStatusesSuccess(response.data, next ? next.uri : null));
+    }).catch(error => {
+      dispatch(fetchBookmarkedStatusesFail(error));
+    });
+  };
+};
+
+export function fetchBookmarkedStatusesRequest() {
+  return {
+    type: BOOKMARKED_STATUSES_FETCH_REQUEST,
+  };
+};
+
+export function fetchBookmarkedStatusesSuccess(statuses, next) {
+  return {
+    type: BOOKMARKED_STATUSES_FETCH_SUCCESS,
+    statuses,
+    next,
+  };
+};
+
+export function fetchBookmarkedStatusesFail(error) {
+  return {
+    type: BOOKMARKED_STATUSES_FETCH_FAIL,
+    error,
+  };
+};
+
+export function expandBookmarkedStatuses() {
+  return (dispatch, getState) => {
+    const url = getState().getIn(['status_lists', 'bookmarks', 'next'], null);
+
+    if (url === null || getState().getIn(['status_lists', 'bookmarks', 'isLoading'])) {
+      return;
+    }
+
+    dispatch(expandBookmarkedStatusesRequest());
+
+    api(getState).get(url).then(response => {
+      const next = getLinks(response).refs.find(link => link.rel === 'next');
+      dispatch(importFetchedStatuses(response.data));
+      dispatch(expandBookmarkedStatusesSuccess(response.data, next ? next.uri : null));
+    }).catch(error => {
+      dispatch(expandBookmarkedStatusesFail(error));
+    });
+  };
+};
+
+export function expandBookmarkedStatusesRequest() {
+  return {
+    type: BOOKMARKED_STATUSES_EXPAND_REQUEST,
+  };
+};
+
+export function expandBookmarkedStatusesSuccess(statuses, next) {
+  return {
+    type: BOOKMARKED_STATUSES_EXPAND_SUCCESS,
+    statuses,
+    next,
+  };
+};
+
+export function expandBookmarkedStatusesFail(error) {
+  return {
+    type: BOOKMARKED_STATUSES_EXPAND_FAIL,
+    error,
+  };
+};
diff --git a/app/javascript/mastodon/actions/interactions.js b/app/javascript/mastodon/actions/interactions.js
index 2dc4c574c..28c6b1a62 100644
--- a/app/javascript/mastodon/actions/interactions.js
+++ b/app/javascript/mastodon/actions/interactions.js
@@ -33,6 +33,14 @@ export const UNPIN_REQUEST = 'UNPIN_REQUEST';
 export const UNPIN_SUCCESS = 'UNPIN_SUCCESS';
 export const UNPIN_FAIL    = 'UNPIN_FAIL';
 
+export const BOOKMARK_REQUEST = 'BOOKMARK_REQUEST';
+export const BOOKMARK_SUCCESS = 'BOOKMARKED_SUCCESS';
+export const BOOKMARK_FAIL    = 'BOOKMARKED_FAIL';
+
+export const UNBOOKMARK_REQUEST = 'UNBOOKMARKED_REQUEST';
+export const UNBOOKMARK_SUCCESS = 'UNBOOKMARKED_SUCCESS';
+export const UNBOOKMARK_FAIL    = 'UNBOOKMARKED_FAIL';
+
 export function reblog(status) {
   return function (dispatch, getState) {
     dispatch(reblogRequest(status));
@@ -187,6 +195,78 @@ export function unfavouriteFail(status, error) {
   };
 };
 
+export function bookmark(status) {
+  return function (dispatch, getState) {
+    dispatch(bookmarkRequest(status));
+
+    api(getState).post(`/api/v1/statuses/${status.get('id')}/bookmark`).then(function (response) {
+      dispatch(importFetchedStatus(response.data));
+      dispatch(bookmarkSuccess(status, response.data));
+    }).catch(function (error) {
+      dispatch(bookmarkFail(status, error));
+    });
+  };
+};
+
+export function unbookmark(status) {
+  return (dispatch, getState) => {
+    dispatch(unbookmarkRequest(status));
+
+    api(getState).post(`/api/v1/statuses/${status.get('id')}/unbookmark`).then(response => {
+      dispatch(importFetchedStatus(response.data));
+      dispatch(unbookmarkSuccess(status, response.data));
+    }).catch(error => {
+      dispatch(unbookmarkFail(status, error));
+    });
+  };
+};
+
+export function bookmarkRequest(status) {
+  return {
+    type: BOOKMARK_REQUEST,
+    status: status,
+  };
+};
+
+export function bookmarkSuccess(status, response) {
+  return {
+    type: BOOKMARK_SUCCESS,
+    status: status,
+    response: response,
+  };
+};
+
+export function bookmarkFail(status, error) {
+  return {
+    type: BOOKMARK_FAIL,
+    status: status,
+    error: error,
+  };
+};
+
+export function unbookmarkRequest(status) {
+  return {
+    type: UNBOOKMARK_REQUEST,
+    status: status,
+  };
+};
+
+export function unbookmarkSuccess(status, response) {
+  return {
+    type: UNBOOKMARK_SUCCESS,
+    status: status,
+    response: response,
+  };
+};
+
+export function unbookmarkFail(status, error) {
+  return {
+    type: UNBOOKMARK_FAIL,
+    status: status,
+    error: error,
+  };
+};
+
 export function fetchReblogs(id) {
   return (dispatch, getState) => {
     dispatch(fetchReblogsRequest(id));
diff --git a/app/javascript/mastodon/components/status_action_bar.js b/app/javascript/mastodon/components/status_action_bar.js
index 0bfbd8879..4fa2c1158 100644
--- a/app/javascript/mastodon/components/status_action_bar.js
+++ b/app/javascript/mastodon/components/status_action_bar.js
@@ -23,6 +23,7 @@ const messages = defineMessages({
   cancel_reblog_private: { id: 'status.cancel_reblog_private', defaultMessage: 'Unboost' },
   cannot_reblog: { id: 'status.cannot_reblog', defaultMessage: 'This post cannot be boosted' },
   favourite: { id: 'status.favourite', defaultMessage: 'Favourite' },
+  bookmark: { id: 'status.bookmark', defaultMessage: 'Bookmark' },
   open: { id: 'status.open', defaultMessage: 'Expand this status' },
   report: { id: 'status.report', defaultMessage: 'Report @{name}' },
   muteConversation: { id: 'status.mute_conversation', defaultMessage: 'Mute conversation' },
@@ -66,6 +67,7 @@ class StatusActionBar extends ImmutablePureComponent {
     onEmbed: PropTypes.func,
     onMuteConversation: PropTypes.func,
     onPin: PropTypes.func,
+    onBookmark: PropTypes.func,
     withDismiss: PropTypes.bool,
     intl: PropTypes.object.isRequired,
   };
@@ -114,6 +116,10 @@ class StatusActionBar extends ImmutablePureComponent {
     window.open(`/interact/${this.props.status.get('id')}?type=${type}`, 'mastodon-intent', 'width=445,height=600,resizable=no,menubar=no,status=no,scrollbars=yes');
   }
 
+  handleBookmarkClick = () => {
+    this.props.onBookmark(this.props.status);
+  }
+
   handleDeleteClick = () => {
     this.props.onDelete(this.props.status, this.context.router.history);
   }
@@ -253,6 +259,7 @@ class StatusActionBar extends ImmutablePureComponent {
         <IconButton className='status__action-bar-button' disabled={!publicStatus} active={status.get('reblogged')} pressed={status.get('reblogged')} title={!publicStatus ? intl.formatMessage(messages.cannot_reblog) : intl.formatMessage(messages.reblog)} icon={reblogIcon} onClick={this.handleReblogClick} />
         <IconButton className='status__action-bar-button star-icon' animate active={status.get('favourited')} pressed={status.get('favourited')} title={intl.formatMessage(messages.favourite)} icon='star' onClick={this.handleFavouriteClick} />
         {shareButton}
+        <IconButton className='status__action-bar-button bookmark-icon' disabled={anonymousAccess} active={status.get('bookmarked')} pressed={status.get('bookmarked')} title={intl.formatMessage(messages.bookmark)} icon='bookmark' onClick={this.handleBookmarkClick} />
 
         <div className='status__action-bar-dropdown'>
           <DropdownMenuContainer disabled={anonymousAccess} status={status} items={menu} icon='ellipsis-h' size={18} direction='right' title={intl.formatMessage(messages.more)} />
diff --git a/app/javascript/mastodon/containers/status_container.js b/app/javascript/mastodon/containers/status_container.js
index fb22676e0..16ba02e12 100644
--- a/app/javascript/mastodon/containers/status_container.js
+++ b/app/javascript/mastodon/containers/status_container.js
@@ -9,8 +9,10 @@ import {
 import {
   reblog,
   favourite,
+  bookmark,
   unreblog,
   unfavourite,
+  unbookmark,
   pin,
   unpin,
 } from '../actions/interactions';
@@ -90,6 +92,14 @@ const mapDispatchToProps = (dispatch, { intl }) => ({
     }
   },
 
+  onBookmark (status) {
+    if (status.get('bookmarked')) {
+      dispatch(unbookmark(status));
+    } else {
+      dispatch(bookmark(status));
+    }
+  },
+
   onPin (status) {
     if (status.get('pinned')) {
       dispatch(unpin(status));
diff --git a/app/javascript/mastodon/features/bookmarked_statuses/index.js b/app/javascript/mastodon/features/bookmarked_statuses/index.js
new file mode 100644
index 000000000..c37cb9176
--- /dev/null
+++ b/app/javascript/mastodon/features/bookmarked_statuses/index.js
@@ -0,0 +1,104 @@
+import React from 'react';
+import { connect } from 'react-redux';
+import PropTypes from 'prop-types';
+import ImmutablePropTypes from 'react-immutable-proptypes';
+import { fetchBookmarkedStatuses, expandBookmarkedStatuses } from '../../actions/bookmarks';
+import Column from '../ui/components/column';
+import ColumnHeader from '../../components/column_header';
+import { addColumn, removeColumn, moveColumn } from '../../actions/columns';
+import StatusList from '../../components/status_list';
+import { defineMessages, injectIntl, FormattedMessage } from 'react-intl';
+import ImmutablePureComponent from 'react-immutable-pure-component';
+import { debounce } from 'lodash';
+
+const messages = defineMessages({
+  heading: { id: 'column.bookmarks', defaultMessage: 'Bookmarks' },
+});
+
+const mapStateToProps = state => ({
+  statusIds: state.getIn(['status_lists', 'bookmarks', 'items']),
+  isLoading: state.getIn(['status_lists', 'bookmarks', 'isLoading'], true),
+  hasMore: !!state.getIn(['status_lists', 'bookmarks', 'next']),
+});
+
+export default @connect(mapStateToProps)
+@injectIntl
+class Bookmarks extends ImmutablePureComponent {
+
+  static propTypes = {
+    dispatch: PropTypes.func.isRequired,
+    shouldUpdateScroll: PropTypes.func,
+    statusIds: ImmutablePropTypes.list.isRequired,
+    intl: PropTypes.object.isRequired,
+    columnId: PropTypes.string,
+    multiColumn: PropTypes.bool,
+    hasMore: PropTypes.bool,
+    isLoading: PropTypes.bool,
+  };
+
+  componentWillMount () {
+    this.props.dispatch(fetchBookmarkedStatuses());
+  }
+
+  handlePin = () => {
+    const { columnId, dispatch } = this.props;
+
+    if (columnId) {
+      dispatch(removeColumn(columnId));
+    } else {
+      dispatch(addColumn('BOOKMARKS', {}));
+    }
+  }
+
+  handleMove = (dir) => {
+    const { columnId, dispatch } = this.props;
+    dispatch(moveColumn(columnId, dir));
+  }
+
+  handleHeaderClick = () => {
+    this.column.scrollTop();
+  }
+
+  setRef = c => {
+    this.column = c;
+  }
+
+  handleLoadMore = debounce(() => {
+    this.props.dispatch(expandBookmarkedStatuses());
+  }, 300, { leading: true })
+
+  render () {
+    const { intl, shouldUpdateScroll, statusIds, columnId, multiColumn, hasMore, isLoading } = this.props;
+    const pinned = !!columnId;
+
+    const emptyMessage = <FormattedMessage id='empty_column.bookmarked_statuses' defaultMessage="You don't have any bookmarked toots yet. When you bookmark one, it will show up here." />;
+
+    return (
+      <Column bindToDocument={!multiColumn} ref={this.setRef} label={intl.formatMessage(messages.heading)}>
+        <ColumnHeader
+          icon='bookmark'
+          title={intl.formatMessage(messages.heading)}
+          onPin={this.handlePin}
+          onMove={this.handleMove}
+          onClick={this.handleHeaderClick}
+          pinned={pinned}
+          multiColumn={multiColumn}
+          showBackButton
+        />
+
+        <StatusList
+          trackScroll={!pinned}
+          statusIds={statusIds}
+          scrollKey={`bookmarked_statuses-${columnId}`}
+          hasMore={hasMore}
+          isLoading={isLoading}
+          onLoadMore={this.handleLoadMore}
+          shouldUpdateScroll={shouldUpdateScroll}
+          emptyMessage={emptyMessage}
+          bindToDocument={!multiColumn}
+        />
+      </Column>
+    );
+  }
+
+}
diff --git a/app/javascript/mastodon/features/getting_started/index.js b/app/javascript/mastodon/features/getting_started/index.js
index 67ec7665b..adbc147d1 100644
--- a/app/javascript/mastodon/features/getting_started/index.js
+++ b/app/javascript/mastodon/features/getting_started/index.js
@@ -22,6 +22,7 @@ const messages = defineMessages({
   settings_subheading: { id: 'column_subheading.settings', defaultMessage: 'Settings' },
   community_timeline: { id: 'navigation_bar.community_timeline', defaultMessage: 'Local timeline' },
   direct: { id: 'navigation_bar.direct', defaultMessage: 'Direct messages' },
+  bookmarks: { id: 'navigation_bar.bookmarks', defaultMessage: 'Bookmarks' },
   preferences: { id: 'navigation_bar.preferences', defaultMessage: 'Preferences' },
   follow_requests: { id: 'navigation_bar.follow_requests', defaultMessage: 'Follow requests' },
   favourites: { id: 'navigation_bar.favourites', defaultMessage: 'Favourites' },
@@ -126,11 +127,12 @@ class GettingStarted extends ImmutablePureComponent {
 
     navItems.push(
       <ColumnLink key={i++} icon='envelope' text={intl.formatMessage(messages.direct)} to='/timelines/direct' />,
+      <ColumnLink key={i++} icon='bookmark' text={intl.formatMessage(messages.bookmarks)} to='/bookmarks' />,
       <ColumnLink key={i++} icon='star' text={intl.formatMessage(messages.favourites)} to='/favourites' />,
       <ColumnLink key={i++} icon='list-ul' text={intl.formatMessage(messages.lists)} to='/lists' />
     );
 
-    height += 48*3;
+    height += 48*4;
 
     if (myAccount.get('locked') || unreadFollowRequests > 0) {
       navItems.push(<ColumnLink key={i++} icon='user-plus' text={intl.formatMessage(messages.follow_requests)} badge={badgeDisplay(unreadFollowRequests, 40)} to='/follow_requests' />);
diff --git a/app/javascript/mastodon/features/status/components/action_bar.js b/app/javascript/mastodon/features/status/components/action_bar.js
index 3e511b7a6..1b81cd245 100644
--- a/app/javascript/mastodon/features/status/components/action_bar.js
+++ b/app/javascript/mastodon/features/status/components/action_bar.js
@@ -17,6 +17,7 @@ const messages = defineMessages({
   cancel_reblog_private: { id: 'status.cancel_reblog_private', defaultMessage: 'Unboost' },
   cannot_reblog: { id: 'status.cannot_reblog', defaultMessage: 'This post cannot be boosted' },
   favourite: { id: 'status.favourite', defaultMessage: 'Favourite' },
+  bookmark: { id: 'status.bookmark', defaultMessage: 'Bookmark' },
   mute: { id: 'status.mute', defaultMessage: 'Mute @{name}' },
   muteConversation: { id: 'status.mute_conversation', defaultMessage: 'Mute conversation' },
   unmuteConversation: { id: 'status.unmute_conversation', defaultMessage: 'Unmute conversation' },
@@ -43,6 +44,7 @@ class ActionBar extends React.PureComponent {
     onReply: PropTypes.func.isRequired,
     onReblog: PropTypes.func.isRequired,
     onFavourite: PropTypes.func.isRequired,
+    onBookmark: PropTypes.func.isRequired,
     onDelete: PropTypes.func.isRequired,
     onDirect: PropTypes.func.isRequired,
     onMention: PropTypes.func.isRequired,
@@ -67,6 +69,10 @@ class ActionBar extends React.PureComponent {
     this.props.onFavourite(this.props.status);
   }
 
+  handleBookmarkClick = (e) => {
+    this.props.onBookmark(this.props.status, e);
+  }
+
   handleDeleteClick = () => {
     this.props.onDelete(this.props.status, this.context.router.history);
   }
@@ -198,6 +204,7 @@ class ActionBar extends React.PureComponent {
         <div className='detailed-status__button'><IconButton disabled={reblog_disabled} active={status.get('reblogged')} title={reblog_disabled ? intl.formatMessage(messages.cannot_reblog) : intl.formatMessage(messages.reblog)} icon={reblogIcon} onClick={this.handleReblogClick} /></div>
         <div className='detailed-status__button'><IconButton className='star-icon' animate active={status.get('favourited')} title={intl.formatMessage(messages.favourite)} icon='star' onClick={this.handleFavouriteClick} /></div>
         {shareButton}
+        <div className='detailed-status__button'><IconButton className='bookmark-icon' active={status.get('bookmarked')} title={intl.formatMessage(messages.bookmark)} icon='bookmark' onClick={this.handleBookmarkClick} /></div>
 
         <div className='detailed-status__action-bar-dropdown'>
           <DropdownMenuContainer size={18} icon='ellipsis-h' items={menu} direction='left' title='More' />
diff --git a/app/javascript/mastodon/features/status/index.js b/app/javascript/mastodon/features/status/index.js
index 029057d40..9fb3fe305 100644
--- a/app/javascript/mastodon/features/status/index.js
+++ b/app/javascript/mastodon/features/status/index.js
@@ -13,6 +13,8 @@ import Column from '../ui/components/column';
 import {
   favourite,
   unfavourite,
+  bookmark,
+  unbookmark,
   reblog,
   unreblog,
   pin,
@@ -232,6 +234,14 @@ class Status extends ImmutablePureComponent {
     }
   }
 
+  handleBookmarkClick = (status) => {
+    if (status.get('bookmarked')) {
+      this.props.dispatch(unbookmark(status));
+    } else {
+      this.props.dispatch(bookmark(status));
+    }
+  }
+
   handleDeleteClick = (status, history, withRedraft = false) => {
     const { dispatch, intl } = this.props;
 
@@ -499,6 +509,7 @@ class Status extends ImmutablePureComponent {
                   onReply={this.handleReplyClick}
                   onFavourite={this.handleFavouriteClick}
                   onReblog={this.handleReblogClick}
+                  onBookmark={this.handleBookmarkClick}
                   onDelete={this.handleDeleteClick}
                   onDirect={this.handleDirectClick}
                   onMention={this.handleMentionClick}
diff --git a/app/javascript/mastodon/features/ui/components/columns_area.js b/app/javascript/mastodon/features/ui/components/columns_area.js
index f31425c70..8bc4bfc0e 100644
--- a/app/javascript/mastodon/features/ui/components/columns_area.js
+++ b/app/javascript/mastodon/features/ui/components/columns_area.js
@@ -21,6 +21,7 @@ import {
   HashtagTimeline,
   DirectTimeline,
   FavouritedStatuses,
+  BookmarkedStatuses,
   ListTimeline,
   Directory,
 } from '../../ui/util/async-components';
@@ -40,6 +41,7 @@ const componentMap = {
   'HASHTAG': HashtagTimeline,
   'DIRECT': DirectTimeline,
   'FAVOURITES': FavouritedStatuses,
+  'BOOKMARKS': BookmarkedStatuses,
   'LIST': ListTimeline,
   'DIRECTORY': Directory,
 };
diff --git a/app/javascript/mastodon/features/ui/components/navigation_panel.js b/app/javascript/mastodon/features/ui/components/navigation_panel.js
index 51e3ec037..0c12852f5 100644
--- a/app/javascript/mastodon/features/ui/components/navigation_panel.js
+++ b/app/javascript/mastodon/features/ui/components/navigation_panel.js
@@ -17,6 +17,7 @@ const NavigationPanel = () => (
     <NavLink className='column-link column-link--transparent' exact to='/timelines/public' data-preview-title-id='column.public' data-preview-icon='globe' ><Icon className='column-link__icon' id='globe' fixedWidth /><FormattedMessage id='tabs_bar.federated_timeline' defaultMessage='Federated' /></NavLink>
     <NavLink className='column-link column-link--transparent' to='/timelines/direct'><Icon className='column-link__icon' id='envelope' fixedWidth /><FormattedMessage id='navigation_bar.direct' defaultMessage='Direct messages' /></NavLink>
     <NavLink className='column-link column-link--transparent' to='/favourites'><Icon className='column-link__icon' id='star' fixedWidth /><FormattedMessage id='navigation_bar.favourites' defaultMessage='Favourites' /></NavLink>
+    <NavLink className='column-link column-link--transparent' to='/bookmarks'><Icon className='column-link__icon' id='bookmark' fixedWidth /><FormattedMessage id='navigation_bar.bookmarks' defaultMessage='Bookmarks' /></NavLink>
     <NavLink className='column-link column-link--transparent' to='/lists'><Icon className='column-link__icon' id='list-ul' fixedWidth /><FormattedMessage id='navigation_bar.lists' defaultMessage='Lists' /></NavLink>
     {profile_directory && <NavLink className='column-link column-link--transparent' to='/directory'><Icon className='column-link__icon' id='address-book-o' fixedWidth /><FormattedMessage id='getting_started.directory' defaultMessage='Profile directory' /></NavLink>}
 
diff --git a/app/javascript/mastodon/features/ui/index.js b/app/javascript/mastodon/features/ui/index.js
index a45ba91eb..b0e38c5cb 100644
--- a/app/javascript/mastodon/features/ui/index.js
+++ b/app/javascript/mastodon/features/ui/index.js
@@ -41,6 +41,7 @@ import {
   FollowRequests,
   GenericNotFound,
   FavouritedStatuses,
+  BookmarkedStatuses,
   ListTimeline,
   Blocks,
   DomainBlocks,
@@ -190,6 +191,7 @@ class SwitchingColumnsArea extends React.PureComponent {
 
           <WrappedRoute path='/notifications' component={Notifications} content={children} componentParams={{ shouldUpdateScroll: this.shouldUpdateScroll }} />
           <WrappedRoute path='/favourites' component={FavouritedStatuses} content={children} componentParams={{ shouldUpdateScroll: this.shouldUpdateScroll }} />
+          <WrappedRoute path='/bookmarks' component={BookmarkedStatuses} content={children} />
           <WrappedRoute path='/pinned' component={PinnedStatuses} content={children} componentParams={{ shouldUpdateScroll: this.shouldUpdateScroll }} />
 
           <WrappedRoute path='/search' component={Search} content={children} />
diff --git a/app/javascript/mastodon/features/ui/util/async-components.js b/app/javascript/mastodon/features/ui/util/async-components.js
index bb0fcb859..986efda1e 100644
--- a/app/javascript/mastodon/features/ui/util/async-components.js
+++ b/app/javascript/mastodon/features/ui/util/async-components.js
@@ -90,6 +90,10 @@ export function FavouritedStatuses () {
   return import(/* webpackChunkName: "features/favourited_statuses" */'../../favourited_statuses');
 }
 
+export function BookmarkedStatuses () {
+  return import(/* webpackChunkName: "features/bookmarked_statuses" */'../../bookmarked_statuses');
+}
+
 export function Blocks () {
   return import(/* webpackChunkName: "features/blocks" */'../../blocks');
 }
diff --git a/app/javascript/mastodon/reducers/status_lists.js b/app/javascript/mastodon/reducers/status_lists.js
index 6c5f33557..9f8f28dee 100644
--- a/app/javascript/mastodon/reducers/status_lists.js
+++ b/app/javascript/mastodon/reducers/status_lists.js
@@ -6,6 +6,14 @@ import {
   FAVOURITED_STATUSES_EXPAND_SUCCESS,
   FAVOURITED_STATUSES_EXPAND_FAIL,
 } from '../actions/favourites';
+import {
+  BOOKMARKED_STATUSES_FETCH_REQUEST,
+  BOOKMARKED_STATUSES_FETCH_SUCCESS,
+  BOOKMARKED_STATUSES_FETCH_FAIL,
+  BOOKMARKED_STATUSES_EXPAND_REQUEST,
+  BOOKMARKED_STATUSES_EXPAND_SUCCESS,
+  BOOKMARKED_STATUSES_EXPAND_FAIL,
+} from '../actions/bookmarks';
 import {
   PINNED_STATUSES_FETCH_SUCCESS,
 } from '../actions/pin_statuses';
@@ -13,6 +21,8 @@ import { Map as ImmutableMap, List as ImmutableList } from 'immutable';
 import {
   FAVOURITE_SUCCESS,
   UNFAVOURITE_SUCCESS,
+  BOOKMARK_SUCCESS,
+  UNBOOKMARK_SUCCESS,
   PIN_SUCCESS,
   UNPIN_SUCCESS,
 } from '../actions/interactions';
@@ -23,6 +33,11 @@ const initialState = ImmutableMap({
     loaded: false,
     items: ImmutableList(),
   }),
+  bookmarks: ImmutableMap({
+    next: null,
+    loaded: false,
+    items: ImmutableList(),
+  }),
   pins: ImmutableMap({
     next: null,
     loaded: false,
@@ -71,10 +86,24 @@ export default function statusLists(state = initialState, action) {
     return normalizeList(state, 'favourites', action.statuses, action.next);
   case FAVOURITED_STATUSES_EXPAND_SUCCESS:
     return appendToList(state, 'favourites', action.statuses, action.next);
+  case BOOKMARKED_STATUSES_FETCH_REQUEST:
+  case BOOKMARKED_STATUSES_EXPAND_REQUEST:
+    return state.setIn(['bookmarks', 'isLoading'], true);
+  case BOOKMARKED_STATUSES_FETCH_FAIL:
+  case BOOKMARKED_STATUSES_EXPAND_FAIL:
+    return state.setIn(['bookmarks', 'isLoading'], false);
+  case BOOKMARKED_STATUSES_FETCH_SUCCESS:
+    return normalizeList(state, 'bookmarks', action.statuses, action.next);
+  case BOOKMARKED_STATUSES_EXPAND_SUCCESS:
+    return appendToList(state, 'bookmarks', action.statuses, action.next);
   case FAVOURITE_SUCCESS:
     return prependOneToList(state, 'favourites', action.status);
   case UNFAVOURITE_SUCCESS:
     return removeOneFromList(state, 'favourites', action.status);
+  case BOOKMARK_SUCCESS:
+    return prependOneToList(state, 'bookmarks', action.status);
+  case UNBOOKMARK_SUCCESS:
+    return removeOneFromList(state, 'bookmarks', action.status);
   case PINNED_STATUSES_FETCH_SUCCESS:
     return normalizeList(state, 'pins', action.statuses, action.next);
   case PIN_SUCCESS:
diff --git a/app/javascript/mastodon/reducers/statuses.js b/app/javascript/mastodon/reducers/statuses.js
index 372673bc0..772f98bcb 100644
--- a/app/javascript/mastodon/reducers/statuses.js
+++ b/app/javascript/mastodon/reducers/statuses.js
@@ -4,6 +4,8 @@ import {
   FAVOURITE_REQUEST,
   FAVOURITE_FAIL,
   UNFAVOURITE_SUCCESS,
+  BOOKMARK_REQUEST,
+  BOOKMARK_FAIL,
 } from '../actions/interactions';
 import {
   STATUS_MUTE_SUCCESS,
@@ -43,6 +45,10 @@ export default function statuses(state = initialState, action) {
     return state.setIn([action.status.get('id'), 'favourites_count'], favouritesCount - 1);
   case FAVOURITE_FAIL:
     return state.get(action.status.get('id')) === undefined ? state : state.setIn([action.status.get('id'), 'favourited'], false);
+  case BOOKMARK_REQUEST:
+    return state.get(action.status.get('id')) === undefined ? state : state.setIn([action.status.get('id'), 'bookmarked'], true);
+  case BOOKMARK_FAIL:
+    return state.get(action.status.get('id')) === undefined ? state : state.setIn([action.status.get('id'), 'bookmarked'], false);
   case REBLOG_REQUEST:
     return state.setIn([action.status.get('id'), 'reblogged'], true);
   case REBLOG_FAIL:
diff --git a/app/javascript/styles/mastodon/components.scss b/app/javascript/styles/mastodon/components.scss
index ded668aea..98ffe1c20 100644
--- a/app/javascript/styles/mastodon/components.scss
+++ b/app/javascript/styles/mastodon/components.scss
@@ -1592,6 +1592,10 @@ a.account__display-name {
   color: $gold-star;
 }
 
+.bookmark-icon.active {
+  color: $red-bookmark;
+}
+
 .no-reduce-motion .icon-button.star-icon {
   &.activate {
     & > .fa-star {
diff --git a/app/javascript/styles/mastodon/variables.scss b/app/javascript/styles/mastodon/variables.scss
index a82c44229..8602c3dde 100644
--- a/app/javascript/styles/mastodon/variables.scss
+++ b/app/javascript/styles/mastodon/variables.scss
@@ -6,6 +6,8 @@ $error-red: #df405a !default;        // Cerise
 $warning-red: #ff5050 !default;      // Sunset Orange
 $gold-star: #ca8f04 !default;        // Dark Goldenrod
 
+$red-bookmark: $warning-red;
+
 // Values from the classic Mastodon UI
 $classic-base-color: #282c37;         // Midnight Express
 $classic-primary-color: #9baec8;      // Echo Blue
diff --git a/app/models/bookmark.rb b/app/models/bookmark.rb
new file mode 100644
index 000000000..01dc48ee7
--- /dev/null
+++ b/app/models/bookmark.rb
@@ -0,0 +1,26 @@
+# frozen_string_literal: true
+# == Schema Information
+#
+# Table name: bookmarks
+#
+#  id         :integer          not null, primary key
+#  created_at :datetime         not null
+#  updated_at :datetime         not null
+#  account_id :integer          not null
+#  status_id  :integer          not null
+#
+
+class Bookmark < ApplicationRecord
+  include Paginable
+
+  update_index('statuses#status', :status) if Chewy.enabled?
+
+  belongs_to :account, inverse_of: :bookmarks
+  belongs_to :status,  inverse_of: :bookmarks
+
+  validates :status_id, uniqueness: { scope: :account_id }
+
+  before_validation do
+    self.status = status.reblog if status&.reblog?
+  end
+end
diff --git a/app/models/concerns/account_associations.rb b/app/models/concerns/account_associations.rb
index c9cc5c610..499edbf4e 100644
--- a/app/models/concerns/account_associations.rb
+++ b/app/models/concerns/account_associations.rb
@@ -13,6 +13,7 @@ module AccountAssociations
     # Timelines
     has_many :statuses, inverse_of: :account, dependent: :destroy
     has_many :favourites, inverse_of: :account, dependent: :destroy
+    has_many :bookmarks, inverse_of: :account, dependent: :destroy
     has_many :mentions, inverse_of: :account, dependent: :destroy
     has_many :notifications, inverse_of: :account, dependent: :destroy
     has_many :conversations, class_name: 'AccountConversation', dependent: :destroy, inverse_of: :account
diff --git a/app/models/concerns/account_interactions.rb b/app/models/concerns/account_interactions.rb
index ad2909d91..f27d39483 100644
--- a/app/models/concerns/account_interactions.rb
+++ b/app/models/concerns/account_interactions.rb
@@ -186,6 +186,10 @@ module AccountInteractions
     status.proper.favourites.where(account: self).exists?
   end
 
+  def bookmarked?(status)
+    status.proper.bookmarks.where(account: self).exists?
+  end
+
   def reblogged?(status)
     status.proper.reblogs.where(account: self).exists?
   end
diff --git a/app/models/status.rb b/app/models/status.rb
index 0c01a5389..1cb381400 100644
--- a/app/models/status.rb
+++ b/app/models/status.rb
@@ -54,6 +54,7 @@ class Status < ApplicationRecord
   belongs_to :reblog, foreign_key: 'reblog_of_id', class_name: 'Status', inverse_of: :reblogs, optional: true
 
   has_many :favourites, inverse_of: :status, dependent: :destroy
+  has_many :bookmarks, inverse_of: :status, dependent: :destroy
   has_many :reblogs, foreign_key: 'reblog_of_id', class_name: 'Status', inverse_of: :reblog, dependent: :destroy
   has_many :replies, foreign_key: 'in_reply_to_id', class_name: 'Status', inverse_of: :thread
   has_many :mentions, dependent: :destroy, inverse_of: :status
@@ -302,6 +303,10 @@ class Status < ApplicationRecord
       Favourite.select('status_id').where(status_id: status_ids).where(account_id: account_id).each_with_object({}) { |f, h| h[f.status_id] = true }
     end
 
+    def bookmarks_map(status_ids, account_id)
+      Bookmark.select('status_id').where(status_id: status_ids).where(account_id: account_id).map { |f| [f.status_id, true] }.to_h
+    end
+
     def reblogs_map(status_ids, account_id)
       unscoped.select('reblog_of_id').where(reblog_of_id: status_ids).where(account_id: account_id).each_with_object({}) { |s, h| h[s.reblog_of_id] = true }
     end
diff --git a/app/presenters/status_relationships_presenter.rb b/app/presenters/status_relationships_presenter.rb
index b04e10e2f..64e688d87 100644
--- a/app/presenters/status_relationships_presenter.rb
+++ b/app/presenters/status_relationships_presenter.rb
@@ -7,6 +7,7 @@ class StatusRelationshipsPresenter
     if current_account_id.nil?
       @reblogs_map    = {}
       @favourites_map = {}
+      @bookmarks_map  = {}
       @mutes_map      = {}
       @pins_map       = {}
     else
@@ -17,6 +18,7 @@ class StatusRelationshipsPresenter
 
       @reblogs_map     = Status.reblogs_map(status_ids, current_account_id).merge(options[:reblogs_map] || {})
       @favourites_map  = Status.favourites_map(status_ids, current_account_id).merge(options[:favourites_map] || {})
+      @bookmarks_map   = Status.bookmarks_map(status_ids, current_account_id).merge(options[:bookmarks_map] || {})
       @mutes_map       = Status.mutes_map(conversation_ids, current_account_id).merge(options[:mutes_map] || {})
       @pins_map        = Status.pins_map(pinnable_status_ids, current_account_id).merge(options[:pins_map] || {})
     end
diff --git a/app/serializers/rest/status_serializer.rb b/app/serializers/rest/status_serializer.rb
index 2dc4a1b61..08bc4d82a 100644
--- a/app/serializers/rest/status_serializer.rb
+++ b/app/serializers/rest/status_serializer.rb
@@ -9,6 +9,7 @@ class REST::StatusSerializer < ActiveModel::Serializer
   attribute :favourited, if: :current_user?
   attribute :reblogged, if: :current_user?
   attribute :muted, if: :current_user?
+  attribute :bookmarked, if: :current_user?
   attribute :pinned, if: :pinnable?
 
   attribute :content, unless: :source_requested?
@@ -93,6 +94,14 @@ class REST::StatusSerializer < ActiveModel::Serializer
     end
   end
 
+  def bookmarked
+    if instance_options && instance_options[:bookmarks]
+      instance_options[:bookmarks].bookmarks_map[object.id] || false
+    else
+      current_user.account.bookmarked?(object)
+    end
+  end
+
   def pinned
     if instance_options && instance_options[:relationships]
       instance_options[:relationships].pins_map[object.id] || false
diff --git a/app/services/backup_service.rb b/app/services/backup_service.rb
index 12e4fa8b4..fe26d7aa0 100644
--- a/app/services/backup_service.rb
+++ b/app/services/backup_service.rb
@@ -45,6 +45,7 @@ class BackupService < BaseService
           dump_media_attachments!(tar)
           dump_outbox!(tar)
           dump_likes!(tar)
+          dump_bookmarks!(tar)
           dump_actor!(tar)
         end
       end
@@ -85,6 +86,7 @@ class BackupService < BaseService
     actor[:image][:url] = 'header' + File.extname(actor[:image][:url]) if actor[:image]
     actor[:outbox]      = 'outbox.json'
     actor[:likes]       = 'likes.json'
+    actor[:bookmarks]   = 'bookmarks.json'
 
     download_to_tar(tar, account.avatar, 'avatar' + File.extname(account.avatar.path)) if account.avatar.exists?
     download_to_tar(tar, account.header, 'header' + File.extname(account.header.path)) if account.header.exists?
@@ -115,6 +117,25 @@ class BackupService < BaseService
     end
   end
 
+  def dump_bookmarks!(tar)
+    collection = serialize(ActivityPub::CollectionPresenter.new(id: 'bookmarks.json', type: :ordered, size: 0, items: []), ActivityPub::CollectionSerializer)
+
+    Status.reorder(nil).joins(:bookmarks).includes(:account).merge(account.bookmarks).find_in_batches do |statuses|
+      statuses.each do |status|
+        collection[:totalItems] += 1
+        collection[:orderedItems] << ActivityPub::TagManager.instance.uri_for(status)
+      end
+
+      GC.start
+    end
+
+    json = Oj.dump(collection)
+
+    tar.add_file_simple('bookmarks.json', 0o444, json.bytesize) do |io|
+      io.write(json)
+    end
+  end
+
   def collection_presenter
     ActivityPub::CollectionPresenter.new(
       id: 'outbox.json',
diff --git a/config/initializers/doorkeeper.rb b/config/initializers/doorkeeper.rb
index 914b3c001..a5c9caa4a 100644
--- a/config/initializers/doorkeeper.rb
+++ b/config/initializers/doorkeeper.rb
@@ -58,6 +58,7 @@ Doorkeeper.configure do
   optional_scopes :write,
                   :'write:accounts',
                   :'write:blocks',
+                  :'write:bookmarks',
                   :'write:conversations',
                   :'write:favourites',
                   :'write:filters',
@@ -71,6 +72,7 @@ Doorkeeper.configure do
                   :read,
                   :'read:accounts',
                   :'read:blocks',
+                  :'read:bookmarks',
                   :'read:favourites',
                   :'read:filters',
                   :'read:follows',
diff --git a/config/locales/doorkeeper.en.yml b/config/locales/doorkeeper.en.yml
index d9b7c2c8e..4e9c83a8f 100644
--- a/config/locales/doorkeeper.en.yml
+++ b/config/locales/doorkeeper.en.yml
@@ -125,6 +125,7 @@ en:
       read: read all your account's data
       read:accounts: see accounts information
       read:blocks: see your blocks
+      read:bookmarks: see your bookmarks
       read:favourites: see your favourites
       read:filters: see your filters
       read:follows: see your follows
@@ -137,6 +138,7 @@ en:
       write: modify all your account's data
       write:accounts: modify your profile
       write:blocks: block accounts and domains
+      write:bookmarks: bookmark statuses
       write:favourites: favourite statuses
       write:filters: create filters
       write:follows: follow people
diff --git a/config/routes.rb b/config/routes.rb
index e43e201a5..5411cff58 100644
--- a/config/routes.rb
+++ b/config/routes.rb
@@ -289,6 +289,9 @@ Rails.application.routes.draw do
           resource :favourite, only: :create
           post :unfavourite, to: 'favourites#destroy'
 
+          resource :bookmark, only: :create
+          post :unbookmark, to: 'bookmarks#destroy'
+
           resource :mute, only: :create
           post :unmute, to: 'mutes#destroy'
 
@@ -324,6 +327,7 @@ Rails.application.routes.draw do
       resources :blocks,       only: [:index]
       resources :mutes,        only: [:index]
       resources :favourites,   only: [:index]
+      resources :bookmarks,    only: [:index]
       resources :reports,      only: [:create]
       resources :trends,       only: [:index]
       resources :filters,      only: [:index, :create, :show, :update, :destroy]
diff --git a/db/migrate/20180831171112_create_bookmarks.rb b/db/migrate/20180831171112_create_bookmarks.rb
new file mode 100644
index 000000000..27c7339c9
--- /dev/null
+++ b/db/migrate/20180831171112_create_bookmarks.rb
@@ -0,0 +1,17 @@
+class CreateBookmarks < ActiveRecord::Migration[5.1]
+  def change
+    create_table :bookmarks do |t|
+      t.references :account, null: false
+      t.references :status, null: false
+
+      t.timestamps
+    end
+
+    safety_assured do
+      add_foreign_key :bookmarks, :accounts, column: :account_id, on_delete: :cascade
+      add_foreign_key :bookmarks, :statuses, column: :status_id, on_delete: :cascade
+    end
+
+    add_index :bookmarks, [:account_id, :status_id], unique: true
+  end
+end
diff --git a/db/schema.rb b/db/schema.rb
index 1bcc003a5..84c76e4ce 100644
--- a/db/schema.rb
+++ b/db/schema.rb
@@ -217,6 +217,16 @@ ActiveRecord::Schema.define(version: 2019_10_31_163205) do
     t.index ["target_account_id"], name: "index_blocks_on_target_account_id"
   end
 
+  create_table "bookmarks", force: :cascade do |t|
+    t.bigint "account_id", null: false
+    t.bigint "status_id", null: false
+    t.datetime "created_at", null: false
+    t.datetime "updated_at", null: false
+    t.index ["account_id", "status_id"], name: "index_bookmarks_on_account_id_and_status_id", unique: true
+    t.index ["account_id"], name: "index_bookmarks_on_account_id"
+    t.index ["status_id"], name: "index_bookmarks_on_status_id"
+  end
+
   create_table "conversation_mutes", force: :cascade do |t|
     t.bigint "conversation_id", null: false
     t.bigint "account_id", null: false
@@ -834,6 +844,8 @@ ActiveRecord::Schema.define(version: 2019_10_31_163205) do
   add_foreign_key "backups", "users", on_delete: :nullify
   add_foreign_key "blocks", "accounts", column: "target_account_id", name: "fk_9571bfabc1", on_delete: :cascade
   add_foreign_key "blocks", "accounts", name: "fk_4269e03e65", on_delete: :cascade
+  add_foreign_key "bookmarks", "accounts", on_delete: :cascade
+  add_foreign_key "bookmarks", "statuses", on_delete: :cascade
   add_foreign_key "conversation_mutes", "accounts", name: "fk_225b4212bb", on_delete: :cascade
   add_foreign_key "conversation_mutes", "conversations", on_delete: :cascade
   add_foreign_key "custom_filters", "accounts", on_delete: :cascade
diff --git a/spec/controllers/api/v1/bookmarks_controller_spec.rb b/spec/controllers/api/v1/bookmarks_controller_spec.rb
new file mode 100644
index 000000000..79601b6e6
--- /dev/null
+++ b/spec/controllers/api/v1/bookmarks_controller_spec.rb
@@ -0,0 +1,78 @@
+require 'rails_helper'
+
+RSpec.describe Api::V1::BookmarksController, type: :controller do
+  render_views
+
+  let(:user)  { Fabricate(:user) }
+  let(:token) { Fabricate(:accessible_access_token, resource_owner_id: user.id, scopes: 'read:bookmarks') }
+
+  describe 'GET #index' do
+    context 'without token' do
+      it 'returns http unauthorized' do
+        get :index
+        expect(response).to have_http_status :unauthorized
+      end
+    end
+
+    context 'with token' do
+      context 'without read scope' do
+        before do
+          allow(controller).to receive(:doorkeeper_token) do
+            Fabricate(:accessible_access_token, resource_owner_id: user.id, scopes: '')
+          end
+        end
+
+        it 'returns http forbidden' do
+          get :index
+          expect(response).to have_http_status :forbidden
+        end
+      end
+
+      context 'without valid resource owner' do
+        before do
+          token = Fabricate(:accessible_access_token, resource_owner_id: user.id, scopes: 'read')
+          user.destroy!
+
+          allow(controller).to receive(:doorkeeper_token) { token }
+        end
+
+        it 'returns http unprocessable entity' do
+          get :index
+          expect(response).to have_http_status :unprocessable_entity
+        end
+      end
+
+      context 'with read scope and valid resource owner' do
+        before do
+          allow(controller).to receive(:doorkeeper_token) do
+            Fabricate(:accessible_access_token, resource_owner_id: user.id, scopes: 'read')
+          end
+        end
+
+        it 'shows bookmarks owned by the user' do
+          bookmarked_by_user = Fabricate(:bookmark, account: user.account)
+          bookmarked_by_others = Fabricate(:bookmark)
+
+          get :index
+
+          expect(assigns(:statuses)).to match_array [bookmarked_by_user.status]
+        end
+
+        it 'adds pagination headers if necessary' do
+          bookmark = Fabricate(:bookmark, account: user.account)
+
+          get :index, params: { limit: 1 }
+
+          expect(response.headers['Link'].find_link(['rel', 'next']).href).to eq "http://test.host/api/v1/bookmarks?limit=1&max_id=#{bookmark.id}"
+          expect(response.headers['Link'].find_link(['rel', 'prev']).href).to eq "http://test.host/api/v1/bookmarks?limit=1&since_id=#{bookmark.id}"
+        end
+
+        it 'does not add pagination headers if not necessary' do
+          get :index
+
+          expect(response.headers['Link']).to eq nil
+        end
+      end
+    end
+  end
+end
diff --git a/spec/controllers/api/v1/statuses/bookmarks_controller_spec.rb b/spec/controllers/api/v1/statuses/bookmarks_controller_spec.rb
new file mode 100644
index 000000000..b79853718
--- /dev/null
+++ b/spec/controllers/api/v1/statuses/bookmarks_controller_spec.rb
@@ -0,0 +1,57 @@
+# frozen_string_literal: true
+
+require 'rails_helper'
+
+describe Api::V1::Statuses::BookmarksController do
+  render_views
+
+  let(:user)  { Fabricate(:user, account: Fabricate(:account, username: 'alice')) }
+  let(:app)   { Fabricate(:application, name: 'Test app', website: 'http://testapp.com') }
+  let(:token) { Fabricate(:accessible_access_token, resource_owner_id: user.id, scopes: 'write:bookmarks', application: app) }
+
+  context 'with an oauth token' do
+    before do
+      allow(controller).to receive(:doorkeeper_token) { token }
+    end
+
+    describe 'POST #create' do
+      let(:status) { Fabricate(:status, account: user.account) }
+
+      before do
+        post :create, params: { status_id: status.id }
+      end
+
+      it 'returns http success' do
+        expect(response).to have_http_status(:success)
+      end
+
+      it 'updates the bookmarked attribute' do
+        expect(user.account.bookmarked?(status)).to be true
+      end
+
+      it 'return json with updated attributes' do
+        hash_body = body_as_json
+
+        expect(hash_body[:id]).to eq status.id.to_s
+        expect(hash_body[:bookmarked]).to be true
+      end
+    end
+
+    describe 'POST #destroy' do
+      let(:status) { Fabricate(:status, account: user.account) }
+
+      before do
+        Bookmark.find_or_create_by!(account: user.account, status: status)
+        post :destroy, params: { status_id: status.id }
+      end
+
+      it 'returns http success' do
+        expect(response).to have_http_status(:success)
+      end
+
+      it 'updates the bookmarked attribute' do
+        expect(user.account.bookmarked?(status)).to be false
+      end
+    end
+  end
+end
diff --git a/spec/fabricators/bookmark_fabricator.rb b/spec/fabricators/bookmark_fabricator.rb
new file mode 100644
index 000000000..12cbc5bfa
--- /dev/null
+++ b/spec/fabricators/bookmark_fabricator.rb
@@ -0,0 +1,4 @@
+Fabricator(:bookmark) do
+  account
+  status
+end
-- 
cgit 


From 510e184216e69aeab3f3f4e421617c640aa7def9 Mon Sep 17 00:00:00 2001
From: Eugen Rochko <eugen@zeonfederated.com>
Date: Fri, 15 Nov 2019 21:00:09 +0100
Subject: Fix localization test failing due to order of locale definitions
 (#12393)

---
 config/application.rb              | 4 ++--
 spec/requests/localization_spec.rb | 1 +
 2 files changed, 3 insertions(+), 2 deletions(-)

(limited to 'spec')

diff --git a/config/application.rb b/config/application.rb
index 6cd660ca2..e1f7ae707 100644
--- a/config/application.rb
+++ b/config/application.rb
@@ -53,8 +53,8 @@ module Mastodon
       :el,
       :en,
       :eo,
-      :es,
       :'es-AR',
+      :es,
       :et,
       :eu,
       :fa,
@@ -93,8 +93,8 @@ module Mastodon
       :sk,
       :sl,
       :sq,
-      :sr,
       :'sr-Latn',
+      :sr,
       :sv,
       :ta,
       :te,
diff --git a/spec/requests/localization_spec.rb b/spec/requests/localization_spec.rb
index 496a885e8..175f02ae9 100644
--- a/spec/requests/localization_spec.rb
+++ b/spec/requests/localization_spec.rb
@@ -26,6 +26,7 @@ describe 'Localization' do
       I18n.t('about.tagline', locale: 'es')
     )
   end
+
   it 'falls back to english when locale is missing' do
     headers = { 'Accept-Language' => '12-FAKE' }
 
-- 
cgit 


From fd93a9c871e0c758603b5a939dc192aeefddab1d Mon Sep 17 00:00:00 2001
From: Jennifer Glauche <=^.^=@github19.jglauche.de>
Date: Sat, 16 Nov 2019 19:02:09 +0100
Subject: make it not return http 400 when passing and empty source argument
 (#12259)

* make it not return http 400 when passing and empty source argument

* create a spec for the empty source hash bug

* compact checks for nil, empty? parameters

* use nil.blank? instead checking for nil
---
 app/controllers/api/v1/accounts/credentials_controller.rb   |  2 +-
 .../api/v1/accounts/credentials_controller_spec.rb          | 13 +++++++++++++
 2 files changed, 14 insertions(+), 1 deletion(-)

(limited to 'spec')

diff --git a/app/controllers/api/v1/accounts/credentials_controller.rb b/app/controllers/api/v1/accounts/credentials_controller.rb
index e77f57910..64b5cb747 100644
--- a/app/controllers/api/v1/accounts/credentials_controller.rb
+++ b/app/controllers/api/v1/accounts/credentials_controller.rb
@@ -25,7 +25,7 @@ class Api::V1::Accounts::CredentialsController < Api::BaseController
   end
 
   def user_settings_params
-    return nil unless params.key?(:source)
+    return nil if params[:source].blank?
 
     source_params = params.require(:source)
 
diff --git a/spec/controllers/api/v1/accounts/credentials_controller_spec.rb b/spec/controllers/api/v1/accounts/credentials_controller_spec.rb
index 19ac32612..ebd462a03 100644
--- a/spec/controllers/api/v1/accounts/credentials_controller_spec.rb
+++ b/spec/controllers/api/v1/accounts/credentials_controller_spec.rb
@@ -59,6 +59,19 @@ describe Api::V1::Accounts::CredentialsController do
         end
       end
 
+      describe 'with empty source list' do
+        before do
+          patch :update, params: {
+            display_name: "I'm a cat",
+            source: {},
+          }, as: :json
+        end
+
+        it 'returns http success' do
+          expect(response).to have_http_status(200)
+        end
+     end
+
       describe 'with invalid data' do
         before do
           patch :update, params: { note: 'This is too long. ' * 30 }
-- 
cgit 


From 5a2c0707f163f14565d186db48bf0b4fe0b05b4f Mon Sep 17 00:00:00 2001
From: Gomasy <nyan@gomasy.jp>
Date: Mon, 18 Nov 2019 01:09:42 +0900
Subject: Support min_id-based pagination for bookmarks (#12381)

* Support min_id-based pagination for bookmarks

* Fix spec
---
 app/controllers/api/v1/bookmarks_controller.rb       | 7 +++----
 spec/controllers/api/v1/bookmarks_controller_spec.rb | 2 +-
 2 files changed, 4 insertions(+), 5 deletions(-)

(limited to 'spec')

diff --git a/app/controllers/api/v1/bookmarks_controller.rb b/app/controllers/api/v1/bookmarks_controller.rb
index cf4cba8dd..e1b244e76 100644
--- a/app/controllers/api/v1/bookmarks_controller.rb
+++ b/app/controllers/api/v1/bookmarks_controller.rb
@@ -26,10 +26,9 @@ class Api::V1::BookmarksController < Api::BaseController
   end
 
   def results
-    @_results ||= account_bookmarks.paginate_by_max_id(
+    @_results ||= account_bookmarks.paginate_by_id(
       limit_param(DEFAULT_STATUSES_LIMIT),
-      params[:max_id],
-      params[:since_id]
+      params_slice(:max_id, :since_id, :min_id)
     )
   end
 
@@ -46,7 +45,7 @@ class Api::V1::BookmarksController < Api::BaseController
   end
 
   def prev_path
-    api_v1_bookmarks_url pagination_params(since_id: pagination_since_id) unless results.empty?
+    api_v1_bookmarks_url pagination_params(min_id: pagination_since_id) unless results.empty?
   end
 
   def pagination_max_id
diff --git a/spec/controllers/api/v1/bookmarks_controller_spec.rb b/spec/controllers/api/v1/bookmarks_controller_spec.rb
index 79601b6e6..d7c5847b0 100644
--- a/spec/controllers/api/v1/bookmarks_controller_spec.rb
+++ b/spec/controllers/api/v1/bookmarks_controller_spec.rb
@@ -64,7 +64,7 @@ RSpec.describe Api::V1::BookmarksController, type: :controller do
           get :index, params: { limit: 1 }
 
           expect(response.headers['Link'].find_link(['rel', 'next']).href).to eq "http://test.host/api/v1/bookmarks?limit=1&max_id=#{bookmark.id}"
-          expect(response.headers['Link'].find_link(['rel', 'prev']).href).to eq "http://test.host/api/v1/bookmarks?limit=1&since_id=#{bookmark.id}"
+          expect(response.headers['Link'].find_link(['rel', 'prev']).href).to eq "http://test.host/api/v1/bookmarks?limit=1&min_id=#{bookmark.id}"
         end
 
         it 'does not add pagination headers if not necessary' do
-- 
cgit 


From d14e74eff52352f1a2fb4bc2053bbb28c1aa29e0 Mon Sep 17 00:00:00 2001
From: Eugen Rochko <eugen@zeonfederated.com>
Date: Sun, 17 Nov 2019 18:40:33 +0100
Subject: Add cache for OEmbed endpoints to avoid extra HTTP requests (#12403)

* add youtube oembed endpoint

* add check for oembed endpoint

* change unless for a more readable if

* clear blank lines

* endpoint via https

* Fix string literal in condition

* use cache for endpoints

* use cache for endpoints

* clean up and adding check

* clean up and remove redundant return

* add html check

* add false to return

* use double quotes

* use double quotes

* Clean up
---
 app/services/fetch_link_card_service.rb    | 23 +++++++++++++++-------
 app/services/fetch_oembed_service.rb       | 31 +++++++++++++++++++++++++++++-
 spec/services/fetch_oembed_service_spec.rb | 18 +++++++++++++++++
 3 files changed, 64 insertions(+), 8 deletions(-)

(limited to 'spec')

diff --git a/app/services/fetch_link_card_service.rb b/app/services/fetch_link_card_service.rb
index 640c60fd4..29880e8d8 100644
--- a/app/services/fetch_link_card_service.rb
+++ b/app/services/fetch_link_card_service.rb
@@ -39,6 +39,12 @@ class FetchLinkCardService < BaseService
   def process_url
     @card ||= PreviewCard.new(url: @url)
 
+    attempt_oembed || attempt_opengraph
+  end
+
+  def html
+    return @html if defined?(@html)
+
     Request.new(:get, @url).perform do |res|
       if res.code == 200 && res.mime_type == 'text/html'
         @html = res.body_with_limit
@@ -48,10 +54,6 @@ class FetchLinkCardService < BaseService
         @html_charset = nil
       end
     end
-
-    return if @html.nil?
-
-    attempt_oembed || attempt_opengraph
   end
 
   def attach_card
@@ -88,12 +90,17 @@ class FetchLinkCardService < BaseService
   end
 
   def attempt_oembed
-    service = FetchOEmbedService.new
-    embed   = service.call(@url, html: @html)
-    url     = Addressable::URI.parse(service.endpoint_url)
+    service         = FetchOEmbedService.new
+    url_domain      = Addressable::URI.parse(@url).normalized_host
+    cached_endpoint = Rails.cache.read("oembed_endpoint:#{url_domain}")
+
+    embed   = service.call(@url, cached_endpoint: cached_endpoint) unless cached_endpoint.nil?
+    embed ||= service.call(@url, html: html) unless html.nil?
 
     return false if embed.nil?
 
+    url = Addressable::URI.parse(service.endpoint_url)
+
     @card.type          = embed[:type]
     @card.title         = embed[:title]         || ''
     @card.author_name   = embed[:author_name]   || ''
@@ -127,6 +134,8 @@ class FetchLinkCardService < BaseService
   end
 
   def attempt_opengraph
+    return if html.nil?
+
     detector = CharlockHolmes::EncodingDetector.new
     detector.strip_tags = true
 
diff --git a/app/services/fetch_oembed_service.rb b/app/services/fetch_oembed_service.rb
index 10176cfb9..4f8498c62 100644
--- a/app/services/fetch_oembed_service.rb
+++ b/app/services/fetch_oembed_service.rb
@@ -1,13 +1,20 @@
 # frozen_string_literal: true
 
 class FetchOEmbedService
+  ENDPOINT_CACHE_EXPIRES_IN = 24.hours.freeze
+
   attr_reader :url, :options, :format, :endpoint_url
 
   def call(url, options = {})
     @url     = url
     @options = options
 
-    discover_endpoint!
+    if @options[:cached_endpoint]
+      parse_cached_endpoint!
+    else
+      discover_endpoint!
+    end
+
     fetch!
   end
 
@@ -32,10 +39,32 @@ class FetchOEmbedService
     return if @endpoint_url.blank?
 
     @endpoint_url = (Addressable::URI.parse(@url) + @endpoint_url).to_s
+
+    cache_endpoint!
   rescue Addressable::URI::InvalidURIError
     @endpoint_url = nil
   end
 
+  def parse_cached_endpoint!
+    cached = @options[:cached_endpoint]
+
+    return if cached[:endpoint].nil? || cached[:format].nil?
+
+    @endpoint_url = Addressable::Template.new(cached[:endpoint]).expand(url: @url).to_s
+    @format       = cached[:format]
+  end
+
+  def cache_endpoint!
+    url_domain = Addressable::URI.parse(@url).normalized_host
+
+    endpoint_hash = {
+      endpoint: @endpoint_url.gsub(URI.encode_www_form_component(@url), '{url}'),
+      format: @format,
+    }
+
+    Rails.cache.write("oembed_endpoint:#{url_domain}", endpoint_hash, expires_in: ENDPOINT_CACHE_EXPIRES_IN)
+  end
+
   def fetch!
     return if @endpoint_url.blank?
 
diff --git a/spec/services/fetch_oembed_service_spec.rb b/spec/services/fetch_oembed_service_spec.rb
index 5789fb53b..a4262b040 100644
--- a/spec/services/fetch_oembed_service_spec.rb
+++ b/spec/services/fetch_oembed_service_spec.rb
@@ -113,6 +113,24 @@ describe FetchOEmbedService, type: :service do
 
     end
 
+    context 'when endpoint is cached' do
+      before do
+        stub_request(:get, 'http://www.youtube.com/oembed?format=json&url=https://www.youtube.com/watch?v=dqwpQarrDwk').to_return(
+          status: 200,
+          headers: { 'Content-Type': 'text/html' },
+          body: request_fixture('oembed_json_empty.html')
+        )
+      end
+
+      it 'returns new provider without fetching original URL first' do
+        subject.call('https://www.youtube.com/watch?v=dqwpQarrDwk', cached_endpoint: { endpoint: 'http://www.youtube.com/oembed?format=json&url={url}', format: :json })
+        expect(a_request(:get, 'https://www.youtube.com/watch?v=dqwpQarrDwk')).to_not have_been_made
+        expect(subject.endpoint_url).to eq 'http://www.youtube.com/oembed?format=json&url=https%3A%2F%2Fwww.youtube.com%2Fwatch%3Fv%3DdqwpQarrDwk'
+        expect(subject.format).to eq :json
+        expect(a_request(:get, 'http://www.youtube.com/oembed?format=json&url=https%3A%2F%2Fwww.youtube.com%2Fwatch%3Fv%3DdqwpQarrDwk')).to have_been_made
+      end
+    end
+
     context 'when status code is not 200' do
       before do
         stub_request(:get, 'https://host.test/oembed.html').to_return(
-- 
cgit 


From cb3e9a293483b0efabd48f025d0b8feed438cc3e Mon Sep 17 00:00:00 2001
From: Dimitri Merejkowsky <dmerejkowsky@users.noreply.github.com>
Date: Tue, 19 Nov 2019 17:22:40 +0100
Subject: Store rspec statuses in .cache/rspec (#12427)

This allows using rspec with `--only-failures`
---
 .cache/.gitkeep     | 0
 spec/spec_helper.rb | 1 +
 2 files changed, 1 insertion(+)
 create mode 100644 .cache/.gitkeep

(limited to 'spec')

diff --git a/.cache/.gitkeep b/.cache/.gitkeep
new file mode 100644
index 000000000..e69de29bb
diff --git a/spec/spec_helper.rb b/spec/spec_helper.rb
index 45ba1bbd9..e57349cc8 100644
--- a/spec/spec_helper.rb
+++ b/spec/spec_helper.rb
@@ -12,6 +12,7 @@ end
 gc_counter = -1
 
 RSpec.configure do |config|
+  config.example_status_persistence_file_path = ".cache/rspec"
   config.expect_with :rspec do |expectations|
     expectations.include_chain_clauses_in_custom_matcher_descriptions = true
   end
-- 
cgit 


From c656cc2191479551806d288bdfb6a30aa35bcb23 Mon Sep 17 00:00:00 2001
From: ThibG <thib@sitedethib.com>
Date: Thu, 21 Nov 2019 16:04:52 +0100
Subject: Fix FetchLinkCardServices crashing on a tags without a target
 (#12159)

* Add test for links without targets

* Fix FetchLinkCardServices crashing on a tags without a target
---
 app/services/fetch_link_card_service.rb       | 2 +-
 spec/services/fetch_link_card_service_spec.rb | 2 +-
 2 files changed, 2 insertions(+), 2 deletions(-)

(limited to 'spec')

diff --git a/app/services/fetch_link_card_service.rb b/app/services/fetch_link_card_service.rb
index 29880e8d8..5d4a7c303 100644
--- a/app/services/fetch_link_card_service.rb
+++ b/app/services/fetch_link_card_service.rb
@@ -67,7 +67,7 @@ class FetchLinkCardService < BaseService
     else
       html  = Nokogiri::HTML(@status.text)
       links = html.css('a')
-      urls  = links.map { |a| Addressable::URI.parse(a['href']).normalize unless skip_link?(a) }.compact
+      urls  = links.map { |a| Addressable::URI.parse(a['href']) unless skip_link?(a) }.compact.map(&:normalize).compact
     end
 
     urls.reject { |uri| bad_url?(uri) }.first
diff --git a/spec/services/fetch_link_card_service_spec.rb b/spec/services/fetch_link_card_service_spec.rb
index 3c8f6f578..8b296cc70 100644
--- a/spec/services/fetch_link_card_service_spec.rb
+++ b/spec/services/fetch_link_card_service_spec.rb
@@ -80,7 +80,7 @@ RSpec.describe FetchLinkCardService, type: :service do
   end
 
   context 'in a remote status' do
-    let(:status) { Fabricate(:status, account: Fabricate(:account, domain: 'example.com'), text: 'Habt ihr ein paar gute Links zu #<span class="tag"><a href="https://quitter.se/tag/wannacry" target="_blank" rel="tag noopener noreferrer" title="https://quitter.se/tag/wannacry">Wannacry</a></span> herumfliegen?   Ich will mal unter <br> <a href="https://github.com/qbi/WannaCry" target="_blank" rel="noopener noreferrer" title="https://github.com/qbi/WannaCry">https://github.com/qbi/WannaCry</a> was sammeln. !<a href="http://sn.jonkman.ca/group/416/id" target="_blank" rel="noopener noreferrer" title="http://sn.jonkman.ca/group/416/id">security</a>&nbsp;') }
+    let(:status) { Fabricate(:status, account: Fabricate(:account, domain: 'example.com'), text: 'Habt ihr ein paar gute Links zu <a>foo</a> #<span class="tag"><a href="https://quitter.se/tag/wannacry" target="_blank" rel="tag noopener noreferrer" title="https://quitter.se/tag/wannacry">Wannacry</a></span> herumfliegen?   Ich will mal unter <br> <a href="https://github.com/qbi/WannaCry" target="_blank" rel="noopener noreferrer" title="https://github.com/qbi/WannaCry">https://github.com/qbi/WannaCry</a> was sammeln. !<a href="http://sn.jonkman.ca/group/416/id" target="_blank" rel="noopener noreferrer" title="http://sn.jonkman.ca/group/416/id">security</a>&nbsp;') }
 
     it 'parses out URLs' do
       expect(a_request(:get, 'https://github.com/qbi/WannaCry')).to have_been_made.at_least_once
-- 
cgit 


From 911cc144815babf83ddf99f2daa3682021d401b8 Mon Sep 17 00:00:00 2001
From: ThibG <thib@sitedethib.com>
Date: Sun, 1 Dec 2019 17:25:29 +0100
Subject: Add follow_request notification type (#12198)

* Add follow_request notification type

The notification type already existed in the backend but was never pushed
to the front-end. This also means translation strings were also available
for the backend, from the notification mailer.

Unlike other notification types, these are off by default, to match what
I remember of Gargron's view on the topic: that follow requests should not
clutter notifications and should instead be reviewed at the user's own
leisure in the dedicated column.

Since follow requests have their own column, I've deemed it unnecessary to
add a specific tab for them in the notification quick filter.

* Show follow request link in single-column if there are pending requests, even if account isn't locked

* Push follow requests from notifications to the follow_requests list

* Offer to accept or reject follow request from the notification

* Redesign follow request notification
---
 .../api/v1/push/subscriptions_controller.rb        |  2 +-
 .../api/web/push_subscriptions_controller.rb       |  3 +-
 app/javascript/mastodon/actions/notifications.js   |  2 +-
 .../notifications/components/column_settings.js    | 11 ++++
 .../notifications/components/follow_request.js     | 59 ++++++++++++++++++++++
 .../notifications/components/notification.js       | 27 +++++++++-
 .../containers/follow_request_container.js         | 26 ++++++++++
 .../ui/components/follow_requests_nav_link.js      | 13 ++---
 app/javascript/mastodon/reducers/notifications.js  | 11 +++-
 .../mastodon/reducers/push_notifications.js        |  1 +
 app/javascript/mastodon/reducers/settings.js       |  3 ++
 app/javascript/mastodon/reducers/user_lists.js     | 11 ++++
 .../mastodon/service_worker/web_push_locales.js    |  1 +
 app/models/notification.rb                         |  8 +--
 app/services/notify_service.rb                     |  2 +-
 spec/models/notification_spec.rb                   | 26 ----------
 16 files changed, 158 insertions(+), 48 deletions(-)
 create mode 100644 app/javascript/mastodon/features/notifications/components/follow_request.js
 create mode 100644 app/javascript/mastodon/features/notifications/containers/follow_request_container.js

(limited to 'spec')

diff --git a/app/controllers/api/v1/push/subscriptions_controller.rb b/app/controllers/api/v1/push/subscriptions_controller.rb
index 1b658f870..1cbc92b93 100644
--- a/app/controllers/api/v1/push/subscriptions_controller.rb
+++ b/app/controllers/api/v1/push/subscriptions_controller.rb
@@ -51,6 +51,6 @@ class Api::V1::Push::SubscriptionsController < Api::BaseController
 
   def data_params
     return {} if params[:data].blank?
-    params.require(:data).permit(alerts: [:follow, :favourite, :reblog, :mention, :poll])
+    params.require(:data).permit(alerts: [:follow, :follow_request, :favourite, :reblog, :mention, :poll])
   end
 end
diff --git a/app/controllers/api/web/push_subscriptions_controller.rb b/app/controllers/api/web/push_subscriptions_controller.rb
index d8153e082..f388b17e5 100644
--- a/app/controllers/api/web/push_subscriptions_controller.rb
+++ b/app/controllers/api/web/push_subscriptions_controller.rb
@@ -19,6 +19,7 @@ class Api::Web::PushSubscriptionsController < Api::Web::BaseController
     data = {
       alerts: {
         follow: alerts_enabled,
+        follow_request: false,
         favourite: alerts_enabled,
         reblog: alerts_enabled,
         mention: alerts_enabled,
@@ -58,6 +59,6 @@ class Api::Web::PushSubscriptionsController < Api::Web::BaseController
   end
 
   def data_params
-    @data_params ||= params.require(:data).permit(alerts: [:follow, :favourite, :reblog, :mention, :poll])
+    @data_params ||= params.require(:data).permit(alerts: [:follow, :follow_request, :favourite, :reblog, :mention, :poll])
   end
 end
diff --git a/app/javascript/mastodon/actions/notifications.js b/app/javascript/mastodon/actions/notifications.js
index 3a92e0224..798f9b37e 100644
--- a/app/javascript/mastodon/actions/notifications.js
+++ b/app/javascript/mastodon/actions/notifications.js
@@ -110,7 +110,7 @@ export function updateNotifications(notification, intlMessages, intlLocale) {
 const excludeTypesFromSettings = state => state.getIn(['settings', 'notifications', 'shows']).filter(enabled => !enabled).keySeq().toJS();
 
 const excludeTypesFromFilter = filter => {
-  const allTypes = ImmutableList(['follow', 'favourite', 'reblog', 'mention', 'poll']);
+  const allTypes = ImmutableList(['follow', 'follow_request', 'favourite', 'reblog', 'mention', 'poll']);
   return allTypes.filterNot(item => item === filter).toJS();
 };
 
diff --git a/app/javascript/mastodon/features/notifications/components/column_settings.js b/app/javascript/mastodon/features/notifications/components/column_settings.js
index 60a86312a..8bd03fbda 100644
--- a/app/javascript/mastodon/features/notifications/components/column_settings.js
+++ b/app/javascript/mastodon/features/notifications/components/column_settings.js
@@ -57,6 +57,17 @@ export default class ColumnSettings extends React.PureComponent {
           </div>
         </div>
 
+        <div role='group' aria-labelledby='notifications-follow-request'>
+          <span id='notifications-follow-request' className='column-settings__section'><FormattedMessage id='notifications.column_settings.follow_request' defaultMessage='New follow requests:' /></span>
+
+          <div className='column-settings__row'>
+            <SettingToggle prefix='notifications_desktop' settings={settings} settingPath={['alerts', 'follow_request']} onChange={onChange} label={alertStr} />
+            {showPushSettings && <SettingToggle prefix='notifications_push' settings={pushSettings} settingPath={['alerts', 'follow_request']} onChange={this.onPushChange} label={pushStr} />}
+            <SettingToggle prefix='notifications' settings={settings} settingPath={['shows', 'follow_request']} onChange={onChange} label={showStr} />
+            <SettingToggle prefix='notifications' settings={settings} settingPath={['sounds', 'follow_request']} onChange={onChange} label={soundStr} />
+          </div>
+        </div>
+
         <div role='group' aria-labelledby='notifications-favourite'>
           <span id='notifications-favourite' className='column-settings__section'><FormattedMessage id='notifications.column_settings.favourite' defaultMessage='Favourites:' /></span>
 
diff --git a/app/javascript/mastodon/features/notifications/components/follow_request.js b/app/javascript/mastodon/features/notifications/components/follow_request.js
new file mode 100644
index 000000000..a80cfb2fa
--- /dev/null
+++ b/app/javascript/mastodon/features/notifications/components/follow_request.js
@@ -0,0 +1,59 @@
+import React, { Fragment } from 'react';
+import ImmutablePropTypes from 'react-immutable-proptypes';
+import PropTypes from 'prop-types';
+import Avatar from 'mastodon/components/avatar';
+import DisplayName from 'mastodon/components/display_name';
+import Permalink from 'mastodon/components/permalink';
+import IconButton from 'mastodon/components/icon_button';
+import { defineMessages, injectIntl } from 'react-intl';
+import ImmutablePureComponent from 'react-immutable-pure-component';
+
+const messages = defineMessages({
+  authorize: { id: 'follow_request.authorize', defaultMessage: 'Authorize' },
+  reject: { id: 'follow_request.reject', defaultMessage: 'Reject' },
+});
+
+export default @injectIntl
+class FollowRequest extends ImmutablePureComponent {
+
+  static propTypes = {
+    account: ImmutablePropTypes.map.isRequired,
+    onAuthorize: PropTypes.func.isRequired,
+    onReject: PropTypes.func.isRequired,
+    intl: PropTypes.object.isRequired,
+  };
+
+  render () {
+    const { intl, hidden, account, onAuthorize, onReject } = this.props;
+
+    if (!account) {
+      return <div />;
+    }
+
+    if (hidden) {
+      return (
+        <Fragment>
+          {account.get('display_name')}
+          {account.get('username')}
+        </Fragment>
+      );
+    }
+
+    return (
+      <div className='account'>
+        <div className='account__wrapper'>
+          <Permalink key={account.get('id')} className='account__display-name' title={account.get('acct')} href={account.get('url')} to={`/accounts/${account.get('id')}`}>
+            <div className='account__avatar-wrapper'><Avatar account={account} size={36} /></div>
+            <DisplayName account={account} />
+          </Permalink>
+
+          <div className='account__relationship'>
+            <IconButton title={intl.formatMessage(messages.authorize)} icon='check' onClick={onAuthorize} />
+            <IconButton title={intl.formatMessage(messages.reject)} icon='times' onClick={onReject} />
+          </div>
+        </div>
+      </div>
+    );
+  }
+
+}
diff --git a/app/javascript/mastodon/features/notifications/components/notification.js b/app/javascript/mastodon/features/notifications/components/notification.js
index 2dea8afa7..74065e5e2 100644
--- a/app/javascript/mastodon/features/notifications/components/notification.js
+++ b/app/javascript/mastodon/features/notifications/components/notification.js
@@ -7,6 +7,7 @@ import ImmutablePureComponent from 'react-immutable-pure-component';
 import { me } from 'mastodon/initial_state';
 import StatusContainer from 'mastodon/containers/status_container';
 import AccountContainer from 'mastodon/containers/account_container';
+import FollowRequestContainer from '../containers/follow_request_container';
 import Icon from 'mastodon/components/icon';
 import Permalink from 'mastodon/components/permalink';
 
@@ -127,7 +128,29 @@ class Notification extends ImmutablePureComponent {
             </span>
           </div>
 
-          <AccountContainer id={account.get('id')} withNote={false} hidden={this.props.hidden} />
+          <AccountContainer id={account.get('id')} hidden={this.props.hidden} />
+        </div>
+      </HotKeys>
+    );
+  }
+
+  renderFollowRequest (notification, account, link) {
+    const { intl } = this.props;
+
+    return (
+      <HotKeys handlers={this.getHandlers()}>
+        <div className='notification notification-follow-request focusable' tabIndex='0' aria-label={notificationForScreenReader(intl, intl.formatMessage({ id: 'notification.follow_request', defaultMessage: '{name} has requested to follow you' }, { name: account.get('acct') }), notification.get('created_at'))}>
+          <div className='notification__message'>
+            <div className='notification__favourite-icon-wrapper'>
+              <Icon id='user' fixedWidth />
+            </div>
+
+            <span title={notification.get('created_at')}>
+              <FormattedMessage id='notification.follow_request' defaultMessage='{name} has requested to follow you' values={{ name: link }} />
+            </span>
+          </div>
+
+          <FollowRequestContainer id={account.get('id')} withNote={false} hidden={this.props.hidden} />
         </div>
       </HotKeys>
     );
@@ -261,6 +284,8 @@ class Notification extends ImmutablePureComponent {
     switch(notification.get('type')) {
     case 'follow':
       return this.renderFollow(notification, account, link);
+    case 'follow_request':
+      return this.renderFollowRequest(notification, account, link);
     case 'mention':
       return this.renderMention(notification);
     case 'favourite':
diff --git a/app/javascript/mastodon/features/notifications/containers/follow_request_container.js b/app/javascript/mastodon/features/notifications/containers/follow_request_container.js
new file mode 100644
index 000000000..f9f6c577e
--- /dev/null
+++ b/app/javascript/mastodon/features/notifications/containers/follow_request_container.js
@@ -0,0 +1,26 @@
+import { connect } from 'react-redux';
+import { makeGetAccount } from 'mastodon/selectors';
+import FollowRequest from '../components/follow_request';
+import { authorizeFollowRequest, rejectFollowRequest } from 'mastodon/actions/accounts';
+
+const makeMapStateToProps = () => {
+  const getAccount = makeGetAccount();
+
+  const mapStateToProps = (state, props) => ({
+    account: getAccount(state, props.id),
+  });
+
+  return mapStateToProps;
+};
+
+const mapDispatchToProps = (dispatch, { id }) => ({
+  onAuthorize () {
+    dispatch(authorizeFollowRequest(id));
+  },
+
+  onReject () {
+    dispatch(rejectFollowRequest(id));
+  },
+});
+
+export default connect(makeMapStateToProps, mapDispatchToProps)(FollowRequest);
diff --git a/app/javascript/mastodon/features/ui/components/follow_requests_nav_link.js b/app/javascript/mastodon/features/ui/components/follow_requests_nav_link.js
index 90c953893..950ed7b27 100644
--- a/app/javascript/mastodon/features/ui/components/follow_requests_nav_link.js
+++ b/app/javascript/mastodon/features/ui/components/follow_requests_nav_link.js
@@ -4,12 +4,10 @@ import { fetchFollowRequests } from 'mastodon/actions/accounts';
 import { connect } from 'react-redux';
 import { NavLink, withRouter } from 'react-router-dom';
 import IconWithBadge from 'mastodon/components/icon_with_badge';
-import { me } from 'mastodon/initial_state';
 import { List as ImmutableList } from 'immutable';
 import { FormattedMessage } from 'react-intl';
 
 const mapStateToProps = state => ({
-  locked: state.getIn(['accounts', me, 'locked']),
   count: state.getIn(['user_lists', 'follow_requests', 'items'], ImmutableList()).size,
 });
 
@@ -19,22 +17,19 @@ class FollowRequestsNavLink extends React.Component {
 
   static propTypes = {
     dispatch: PropTypes.func.isRequired,
-    locked: PropTypes.bool,
     count: PropTypes.number.isRequired,
   };
 
   componentDidMount () {
-    const { dispatch, locked } = this.props;
+    const { dispatch } = this.props;
 
-    if (locked) {
-      dispatch(fetchFollowRequests());
-    }
+    dispatch(fetchFollowRequests());
   }
 
   render () {
-    const { locked, count } = this.props;
+    const { count } = this.props;
 
-    if (!locked || count === 0) {
+    if (count === 0) {
       return null;
     }
 
diff --git a/app/javascript/mastodon/reducers/notifications.js b/app/javascript/mastodon/reducers/notifications.js
index 6ba80bd6a..60e901e39 100644
--- a/app/javascript/mastodon/reducers/notifications.js
+++ b/app/javascript/mastodon/reducers/notifications.js
@@ -13,6 +13,8 @@ import {
 import {
   ACCOUNT_BLOCK_SUCCESS,
   ACCOUNT_MUTE_SUCCESS,
+  FOLLOW_REQUEST_AUTHORIZE_SUCCESS,
+  FOLLOW_REQUEST_REJECT_SUCCESS,
 } from '../actions/accounts';
 import { DOMAIN_BLOCK_SUCCESS } from 'mastodon/actions/domain_blocks';
 import { TIMELINE_DELETE, TIMELINE_DISCONNECT } from '../actions/timelines';
@@ -89,8 +91,8 @@ const expandNormalizedNotifications = (state, notifications, next, isLoadingRece
   });
 };
 
-const filterNotifications = (state, accountIds) => {
-  const helper = list => list.filterNot(item => item !== null && accountIds.includes(item.get('account')));
+const filterNotifications = (state, accountIds, type) => {
+  const helper = list => list.filterNot(item => item !== null && accountIds.includes(item.get('account')) && (type === undefined || type === item.get('type')));
   return state.update('items', helper).update('pendingItems', helper);
 };
 
@@ -129,6 +131,11 @@ export default function notifications(state = initialState, action) {
     return action.relationship.muting_notifications ? filterNotifications(state, [action.relationship.id]) : state;
   case DOMAIN_BLOCK_SUCCESS:
     return filterNotifications(state, action.accounts);
+  case FOLLOW_REQUEST_AUTHORIZE_SUCCESS:
+  case FOLLOW_REQUEST_REJECT_SUCCESS:
+    return filterNotifications(state, [action.id], 'follow_request');
+  case ACCOUNT_MUTE_SUCCESS:
+    return action.relationship.muting_notifications ? filterNotifications(state, [action.relationship.id]) : state;
   case NOTIFICATIONS_CLEAR:
     return state.set('items', ImmutableList()).set('pendingItems', ImmutableList()).set('hasMore', false);
   case TIMELINE_DELETE:
diff --git a/app/javascript/mastodon/reducers/push_notifications.js b/app/javascript/mastodon/reducers/push_notifications.js
index 317352b79..c48cfb705 100644
--- a/app/javascript/mastodon/reducers/push_notifications.js
+++ b/app/javascript/mastodon/reducers/push_notifications.js
@@ -6,6 +6,7 @@ const initialState = Immutable.Map({
   subscription: null,
   alerts: new Immutable.Map({
     follow: false,
+    follow_request: false,
     favourite: false,
     reblog: false,
     mention: false,
diff --git a/app/javascript/mastodon/reducers/settings.js b/app/javascript/mastodon/reducers/settings.js
index 793a99f8f..efef2ad9a 100644
--- a/app/javascript/mastodon/reducers/settings.js
+++ b/app/javascript/mastodon/reducers/settings.js
@@ -30,6 +30,7 @@ const initialState = ImmutableMap({
   notifications: ImmutableMap({
     alerts: ImmutableMap({
       follow: true,
+      follow_request: false,
       favourite: true,
       reblog: true,
       mention: true,
@@ -44,6 +45,7 @@ const initialState = ImmutableMap({
 
     shows: ImmutableMap({
       follow: true,
+      follow_request: false,
       favourite: true,
       reblog: true,
       mention: true,
@@ -52,6 +54,7 @@ const initialState = ImmutableMap({
 
     sounds: ImmutableMap({
       follow: true,
+      follow_request: false,
       favourite: true,
       reblog: true,
       mention: true,
diff --git a/app/javascript/mastodon/reducers/user_lists.js b/app/javascript/mastodon/reducers/user_lists.js
index 08e94022f..a7853452f 100644
--- a/app/javascript/mastodon/reducers/user_lists.js
+++ b/app/javascript/mastodon/reducers/user_lists.js
@@ -1,3 +1,6 @@
+import {
+  NOTIFICATIONS_UPDATE,
+} from '../actions/notifications';
 import {
   FOLLOWERS_FETCH_SUCCESS,
   FOLLOWERS_EXPAND_SUCCESS,
@@ -53,6 +56,12 @@ const appendToList = (state, type, id, accounts, next) => {
   });
 };
 
+const normalizeFollowRequest = (state, notification) => {
+  return state.updateIn(['follow_requests', 'items'], list => {
+    return list.filterNot(item => item === notification.account.id).unshift(notification.account.id);
+  });
+};
+
 export default function userLists(state = initialState, action) {
   switch(action.type) {
   case FOLLOWERS_FETCH_SUCCESS:
@@ -67,6 +76,8 @@ export default function userLists(state = initialState, action) {
     return state.setIn(['reblogged_by', action.id], ImmutableList(action.accounts.map(item => item.id)));
   case FAVOURITES_FETCH_SUCCESS:
     return state.setIn(['favourited_by', action.id], ImmutableList(action.accounts.map(item => item.id)));
+  case NOTIFICATIONS_UPDATE:
+    return action.notification.type === 'follow_request' ? normalizeFollowRequest(state, action.notification) : state;
   case FOLLOW_REQUESTS_FETCH_SUCCESS:
     return state.setIn(['follow_requests', 'items'], ImmutableList(action.accounts.map(item => item.id))).setIn(['follow_requests', 'next'], action.next);
   case FOLLOW_REQUESTS_EXPAND_SUCCESS:
diff --git a/app/javascript/mastodon/service_worker/web_push_locales.js b/app/javascript/mastodon/service_worker/web_push_locales.js
index 5ce8c7b50..1265f3cfa 100644
--- a/app/javascript/mastodon/service_worker/web_push_locales.js
+++ b/app/javascript/mastodon/service_worker/web_push_locales.js
@@ -16,6 +16,7 @@ filenames.forEach(filename => {
   filtered[locale] = {
     'notification.favourite': full['notification.favourite'] || '',
     'notification.follow': full['notification.follow'] || '',
+    'notification.follow_request': full['notification.follow_request'] || '',
     'notification.mention': full['notification.mention'] || '',
     'notification.reblog': full['notification.reblog'] || '',
     'notification.poll': full['notification.poll'] || '',
diff --git a/app/models/notification.rb b/app/models/notification.rb
index 498673ff1..ad7528f50 100644
--- a/app/models/notification.rb
+++ b/app/models/notification.rb
@@ -42,7 +42,7 @@ class Notification < ApplicationRecord
   validates :activity_type, inclusion: { in: TYPE_CLASS_MAP.values }
 
   scope :browserable, ->(exclude_types = [], account_id = nil) {
-    types = TYPE_CLASS_MAP.values - activity_types_from_types(exclude_types + [:follow_request])
+    types = TYPE_CLASS_MAP.values - activity_types_from_types(exclude_types)
     if account_id.nil?
       where(activity_type: types)
     else
@@ -50,7 +50,7 @@ class Notification < ApplicationRecord
     end
   }
 
-  cache_associated :from_account, status: STATUS_INCLUDES, mention: [status: STATUS_INCLUDES], favourite: [:account, status: STATUS_INCLUDES], follow: :account, poll: [status: STATUS_INCLUDES]
+  cache_associated :from_account, status: STATUS_INCLUDES, mention: [status: STATUS_INCLUDES], favourite: [:account, status: STATUS_INCLUDES], follow: :account, follow_request: :account, poll: [status: STATUS_INCLUDES]
 
   def type
     @type ||= TYPE_CLASS_MAP.invert[activity_type].to_sym
@@ -69,10 +69,6 @@ class Notification < ApplicationRecord
     end
   end
 
-  def browserable?
-    type != :follow_request
-  end
-
   class << self
     def cache_ids
       select(:id, :updated_at, :activity_type, :activity_id)
diff --git a/app/services/notify_service.rb b/app/services/notify_service.rb
index b5c721589..9364a6ae8 100644
--- a/app/services/notify_service.rb
+++ b/app/services/notify_service.rb
@@ -9,7 +9,7 @@ class NotifyService < BaseService
     return if recipient.user.nil? || blocked?
 
     create_notification!
-    push_notification! if @notification.browserable?
+    push_notification!
     push_to_conversation! if direct_message?
     send_email! if email_enabled?
   rescue ActiveRecord::RecordInvalid
diff --git a/spec/models/notification_spec.rb b/spec/models/notification_spec.rb
index 59c582cde..d2e676ec2 100644
--- a/spec/models/notification_spec.rb
+++ b/spec/models/notification_spec.rb
@@ -34,32 +34,6 @@ RSpec.describe Notification, type: :model do
     end
   end
 
-  describe '#browserable?' do
-    let(:notification) { Fabricate(:notification) }
-
-    subject { notification.browserable? }
-
-    context 'type is :follow_request' do
-      before do
-        allow(notification).to receive(:type).and_return(:follow_request)
-      end
-
-      it 'returns false' do
-        is_expected.to be false
-      end
-    end
-
-    context 'type is not :follow_request' do
-      before do
-        allow(notification).to receive(:type).and_return(:else)
-      end
-
-      it 'returns true' do
-        is_expected.to be true
-      end
-    end
-  end
-
   describe '#type' do
     it 'returns :reblog for a Status' do
       notification = Notification.new(activity: Status.new)
-- 
cgit 


From e598ce0bd9dea9b8e0f9fe26a1d78b8ca4838e22 Mon Sep 17 00:00:00 2001
From: Yamagishi Kazutoshi <ykzts@desire.sh>
Date: Tue, 3 Dec 2019 03:55:08 +0900
Subject: Move rspec examples to tmp dir (#12539)

---
 .cache/.gitkeep     | 0
 spec/spec_helper.rb | 2 +-
 2 files changed, 1 insertion(+), 1 deletion(-)
 delete mode 100644 .cache/.gitkeep

(limited to 'spec')

diff --git a/.cache/.gitkeep b/.cache/.gitkeep
deleted file mode 100644
index e69de29bb..000000000
diff --git a/spec/spec_helper.rb b/spec/spec_helper.rb
index e57349cc8..b6d127a08 100644
--- a/spec/spec_helper.rb
+++ b/spec/spec_helper.rb
@@ -12,7 +12,7 @@ end
 gc_counter = -1
 
 RSpec.configure do |config|
-  config.example_status_persistence_file_path = ".cache/rspec"
+  config.example_status_persistence_file_path = "tmp/rspec/examples.txt"
   config.expect_with :rspec do |expectations|
     expectations.include_chain_clauses_in_custom_matcher_descriptions = true
   end
-- 
cgit