From 17747e2cd72c9142c508b6de64717c21d71dc189 Mon Sep 17 00:00:00 2001
From: ThibG <thib@sitedethib.com>
Date: Tue, 18 Jun 2019 18:22:02 +0200
Subject: Fix User#active scope only returning suspended users (#11111)

Fix a regression from #10660
---
 app/models/user.rb | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

(limited to 'app/models')

diff --git a/app/models/user.rb b/app/models/user.rb
index eb1a2fece..4abf124fc 100644
--- a/app/models/user.rb
+++ b/app/models/user.rb
@@ -88,7 +88,7 @@ class User < ApplicationRecord
   scope :confirmed, -> { where.not(confirmed_at: nil) }
   scope :enabled, -> { where(disabled: false) }
   scope :inactive, -> { where(arel_table[:current_sign_in_at].lt(ACTIVE_DURATION.ago)) }
-  scope :active, -> { confirmed.where(arel_table[:current_sign_in_at].gteq(ACTIVE_DURATION.ago)).joins(:account).where.not(accounts: { suspended_at: nil }) }
+  scope :active, -> { confirmed.where(arel_table[:current_sign_in_at].gteq(ACTIVE_DURATION.ago)).joins(:account).where(accounts: { suspended_at: nil }) }
   scope :matches_email, ->(value) { where(arel_table[:email].matches("#{value}%")) }
   scope :emailable, -> { confirmed.enabled.joins(:account).merge(Account.searchable) }
 
-- 
cgit 


From f7f23b4a19a84371f44ec5297125e96ba81681a1 Mon Sep 17 00:00:00 2001
From: Eugen Rochko <eugen@zeonfederated.com>
Date: Wed, 19 Jun 2019 23:42:38 +0200
Subject: Add audio uploads (#11123)

* Add audio uploads

Fix #4827

Accept uploads of OGG, WAV, FLAC, OPUS and MP3 files, and converts
them to OGG. Media attachments get a new `audio` type. In the UI,
audio uploads are displayed identically to video uploads.

* Improve code style
---
 app/controllers/media_controller.rb                | 12 +++-
 app/javascript/mastodon/components/status.js       | 12 ++--
 .../features/compose/components/upload_button.js   |  8 ++-
 .../compose/containers/upload_button_container.js  |  2 +-
 .../features/status/components/detailed_status.js  | 12 ++--
 app/javascript/mastodon/locales/en.json            |  2 +-
 app/models/media_attachment.rb                     | 64 +++++++++++++++-------
 app/serializers/initial_state_serializer.rb        |  2 +-
 app/services/post_status_service.rb                |  2 +-
 .../stream_entries/_detailed_status.html.haml      |  2 +-
 app/views/stream_entries/_simple_status.html.haml  |  2 +-
 11 files changed, 76 insertions(+), 44 deletions(-)

(limited to 'app/models')

diff --git a/app/controllers/media_controller.rb b/app/controllers/media_controller.rb
index a245db2d1..d44b52d26 100644
--- a/app/controllers/media_controller.rb
+++ b/app/controllers/media_controller.rb
@@ -7,6 +7,8 @@ class MediaController < ApplicationController
 
   before_action :set_media_attachment
   before_action :verify_permitted_status!
+  before_action :check_playable, only: :player
+  before_action :allow_iframing, only: :player
 
   content_security_policy only: :player do |p|
     p.frame_ancestors(false)
@@ -18,8 +20,6 @@ class MediaController < ApplicationController
 
   def player
     @body_classes = 'player'
-    response.headers['X-Frame-Options'] = 'ALLOWALL'
-    raise ActiveRecord::RecordNotFound unless @media_attachment.video? || @media_attachment.gifv?
   end
 
   private
@@ -34,4 +34,12 @@ class MediaController < ApplicationController
     # Reraise in order to get a 404 instead of a 403 error code
     raise ActiveRecord::RecordNotFound
   end
+
+  def check_playable
+    not_found unless @media_attachment.larger_media_format?
+  end
+
+  def allow_iframing
+    response.headers['X-Frame-Options'] = 'ALLOWALL'
+  end
 end
diff --git a/app/javascript/mastodon/components/status.js b/app/javascript/mastodon/components/status.js
index aa5e870dc..9b1035649 100644
--- a/app/javascript/mastodon/components/status.js
+++ b/app/javascript/mastodon/components/status.js
@@ -333,17 +333,17 @@ class Status extends ImmutablePureComponent {
             media={status.get('media_attachments')}
           />
         );
-      } else if (status.getIn(['media_attachments', 0, 'type']) === 'video') {
-        const video = status.getIn(['media_attachments', 0]);
+      } else if (['video', 'audio'].includes(status.getIn(['media_attachments', 0, 'type']))) {
+        const attachment = status.getIn(['media_attachments', 0]);
 
         media = (
           <Bundle fetchComponent={Video} loading={this.renderLoadingVideoPlayer} >
             {Component => (
               <Component
-                preview={video.get('preview_url')}
-                blurhash={video.get('blurhash')}
-                src={video.get('url')}
-                alt={video.get('description')}
+                preview={attachment.get('preview_url')}
+                blurhash={attachment.get('blurhash')}
+                src={attachment.get('url')}
+                alt={attachment.get('description')}
                 width={this.props.cachedMediaWidth}
                 height={110}
                 inline
diff --git a/app/javascript/mastodon/features/compose/components/upload_button.js b/app/javascript/mastodon/features/compose/components/upload_button.js
index 90e2769f3..e141857be 100644
--- a/app/javascript/mastodon/features/compose/components/upload_button.js
+++ b/app/javascript/mastodon/features/compose/components/upload_button.js
@@ -7,9 +7,11 @@ import ImmutablePureComponent from 'react-immutable-pure-component';
 import ImmutablePropTypes from 'react-immutable-proptypes';
 
 const messages = defineMessages({
-  upload: { id: 'upload_button.label', defaultMessage: 'Add media (JPEG, PNG, GIF, WebM, MP4, MOV)' },
+  upload: { id: 'upload_button.label', defaultMessage: 'Add media ({formats})' },
 });
 
+const SUPPORTED_FORMATS = 'JPEG, PNG, GIF, WebM, MP4, MOV, OGG, WAV, MP3, FLAC';
+
 const makeMapStateToProps = () => {
   const mapStateToProps = state => ({
     acceptContentTypes: state.getIn(['media_attachments', 'accept_content_types']),
@@ -60,9 +62,9 @@ class UploadButton extends ImmutablePureComponent {
 
     return (
       <div className='compose-form__upload-button'>
-        <IconButton icon='camera' title={intl.formatMessage(messages.upload)} disabled={disabled} onClick={this.handleClick} className='compose-form__upload-button-icon' size={18} inverted style={iconStyle} />
+        <IconButton icon='camera' title={intl.formatMessage(messages.upload, { formats: SUPPORTED_FORMATS })} disabled={disabled} onClick={this.handleClick} className='compose-form__upload-button-icon' size={18} inverted style={iconStyle} />
         <label>
-          <span style={{ display: 'none' }}>{intl.formatMessage(messages.upload)}</span>
+          <span style={{ display: 'none' }}>{intl.formatMessage(messages.upload, { formats: SUPPORTED_FORMATS })}</span>
           <input
             key={resetFileKey}
             ref={this.setRef}
diff --git a/app/javascript/mastodon/features/compose/containers/upload_button_container.js b/app/javascript/mastodon/features/compose/containers/upload_button_container.js
index d8b8c4b6e..1471e628b 100644
--- a/app/javascript/mastodon/features/compose/containers/upload_button_container.js
+++ b/app/javascript/mastodon/features/compose/containers/upload_button_container.js
@@ -3,7 +3,7 @@ import UploadButton from '../components/upload_button';
 import { uploadCompose } from '../../../actions/compose';
 
 const mapStateToProps = state => ({
-  disabled: state.getIn(['compose', 'is_uploading']) || (state.getIn(['compose', 'media_attachments']).size > 3 || state.getIn(['compose', 'media_attachments']).some(m => m.get('type') === 'video')),
+  disabled: state.getIn(['compose', 'is_uploading']) || (state.getIn(['compose', 'media_attachments']).size > 3 || state.getIn(['compose', 'media_attachments']).some(m => ['video', 'audio'].includes(m.get('type')))),
   unavailable: state.getIn(['compose', 'poll']) !== null,
   resetFileKey: state.getIn(['compose', 'resetFileKey']),
 });
diff --git a/app/javascript/mastodon/features/status/components/detailed_status.js b/app/javascript/mastodon/features/status/components/detailed_status.js
index c7aa4d033..4af157af1 100644
--- a/app/javascript/mastodon/features/status/components/detailed_status.js
+++ b/app/javascript/mastodon/features/status/components/detailed_status.js
@@ -107,15 +107,15 @@ export default class DetailedStatus extends ImmutablePureComponent {
     }
 
     if (status.get('media_attachments').size > 0) {
-      if (status.getIn(['media_attachments', 0, 'type']) === 'video') {
-        const video = status.getIn(['media_attachments', 0]);
+      if (['video', 'audio'].includes(status.getIn(['media_attachments', 0, 'type']))) {
+        const attachment = status.getIn(['media_attachments', 0]);
 
         media = (
           <Video
-            preview={video.get('preview_url')}
-            blurhash={video.get('blurhash')}
-            src={video.get('url')}
-            alt={video.get('description')}
+            preview={attachment.get('preview_url')}
+            blurhash={attachment.get('blurhash')}
+            src={attachment.get('url')}
+            alt={attachment.get('description')}
             width={300}
             height={150}
             inline
diff --git a/app/javascript/mastodon/locales/en.json b/app/javascript/mastodon/locales/en.json
index 34ec1ca23..67e1e01f8 100644
--- a/app/javascript/mastodon/locales/en.json
+++ b/app/javascript/mastodon/locales/en.json
@@ -369,7 +369,7 @@
   "trends.count_by_accounts": "{count} {rawCount, plural, one {person} other {people}} talking",
   "ui.beforeunload": "Your draft will be lost if you leave Mastodon.",
   "upload_area.title": "Drag & drop to upload",
-  "upload_button.label": "Add media (JPEG, PNG, GIF, WebM, MP4, MOV)",
+  "upload_button.label": "Add media ({formats})",
   "upload_error.limit": "File upload limit exceeded.",
   "upload_error.poll": "File upload not allowed with polls.",
   "upload_form.description": "Describe for the visually impaired",
diff --git a/app/models/media_attachment.rb b/app/models/media_attachment.rb
index ab794faa0..5ca218411 100644
--- a/app/models/media_attachment.rb
+++ b/app/models/media_attachment.rb
@@ -24,14 +24,16 @@
 class MediaAttachment < ApplicationRecord
   self.inheritance_column = nil
 
-  enum type: [:image, :gifv, :video, :unknown]
+  enum type: [:image, :gifv, :video, :unknown, :audio]
 
   IMAGE_FILE_EXTENSIONS = ['.jpg', '.jpeg', '.png', '.gif', '.webp'].freeze
   VIDEO_FILE_EXTENSIONS = ['.webm', '.mp4', '.m4v', '.mov'].freeze
+  AUDIO_FILE_EXTENSIONS = ['.ogg', '.oga', '.mp3', '.wav', '.flac', '.opus'].freeze
 
   IMAGE_MIME_TYPES             = ['image/jpeg', 'image/png', 'image/gif', 'image/webp'].freeze
   VIDEO_MIME_TYPES             = ['video/webm', 'video/mp4', 'video/quicktime'].freeze
   VIDEO_CONVERTIBLE_MIME_TYPES = ['video/webm', 'video/quicktime'].freeze
+  AUDIO_MIME_TYPES             = ['audio/wave', 'audio/wav', 'audio/x-wav', 'audio/x-pn-wave', 'audio/ogg', 'audio/mpeg', 'audio/webm', 'audio/flac'].freeze
 
   BLURHASH_OPTIONS = {
     x_comp: 4,
@@ -65,6 +67,13 @@ class MediaAttachment < ApplicationRecord
     },
   }.freeze
 
+  AUDIO_STYLES = {
+    original: {
+      format: 'ogg',
+      convert_options: {},
+    },
+  }.freeze
+
   VIDEO_FORMAT = {
     format: 'mp4',
     convert_options: {
@@ -83,6 +92,11 @@ class MediaAttachment < ApplicationRecord
     },
   }.freeze
 
+  VIDEO_CONVERTED_STYLES = {
+    small: VIDEO_STYLES[:small],
+    original: VIDEO_FORMAT,
+  }.freeze
+
   IMAGE_LIMIT = 8.megabytes
   VIDEO_LIMIT = 40.megabytes
 
@@ -95,9 +109,9 @@ class MediaAttachment < ApplicationRecord
                     processors: ->(f) { file_processors f },
                     convert_options: { all: '-quality 90 -strip' }
 
-  validates_attachment_content_type :file, content_type: IMAGE_MIME_TYPES + VIDEO_MIME_TYPES
-  validates_attachment_size :file, less_than: IMAGE_LIMIT, unless: :video_or_gifv?
-  validates_attachment_size :file, less_than: VIDEO_LIMIT, if: :video_or_gifv?
+  validates_attachment_content_type :file, content_type: IMAGE_MIME_TYPES + VIDEO_MIME_TYPES + AUDIO_MIME_TYPES
+  validates_attachment_size :file, less_than: IMAGE_LIMIT, unless: :larger_media_format?
+  validates_attachment_size :file, less_than: VIDEO_LIMIT, if: :larger_media_format?
   remotable_attachment :file, VIDEO_LIMIT
 
   include Attachmentable
@@ -120,8 +134,12 @@ class MediaAttachment < ApplicationRecord
     file.blank? && remote_url.present?
   end
 
-  def video_or_gifv?
-    video? || gifv?
+  def larger_media_format?
+    video? || gifv? || audio?
+  end
+
+  def audio_or_video?
+    audio? || video?
   end
 
   def to_param
@@ -156,28 +174,24 @@ class MediaAttachment < ApplicationRecord
     private
 
     def file_styles(f)
-      if f.instance.file_content_type == 'image/gif'
-        {
-          small: IMAGE_STYLES[:small],
-          original: VIDEO_FORMAT,
-        }
-      elsif IMAGE_MIME_TYPES.include? f.instance.file_content_type
+      if f.instance.file_content_type == 'image/gif' || VIDEO_CONVERTIBLE_MIME_TYPES.include?(f.instance.file_content_type)
+        VIDEO_CONVERTED_STYLES
+      elsif IMAGE_MIME_TYPES.include?(f.instance.file_content_type)
         IMAGE_STYLES
-      elsif VIDEO_CONVERTIBLE_MIME_TYPES.include?(f.instance.file_content_type)
-        {
-          small: VIDEO_STYLES[:small],
-          original: VIDEO_FORMAT,
-        }
-      else
+      elsif VIDEO_MIME_TYPES.include?(f.instance.file_content_type)
         VIDEO_STYLES
+      else
+        AUDIO_STYLES
       end
     end
 
     def file_processors(f)
       if f.file_content_type == 'image/gif'
         [:gif_transcoder, :blurhash_transcoder]
-      elsif VIDEO_MIME_TYPES.include? f.file_content_type
+      elsif VIDEO_MIME_TYPES.include?(f.file_content_type)
         [:video_transcoder, :blurhash_transcoder]
+      elsif AUDIO_MIME_TYPES.include?(f.file_content_type)
+        [:transcoder]
       else
         [:lazy_thumbnail, :blurhash_transcoder]
       end
@@ -202,7 +216,15 @@ class MediaAttachment < ApplicationRecord
   end
 
   def set_type_and_extension
-    self.type = VIDEO_MIME_TYPES.include?(file_content_type) ? :video : :image
+    self.type = begin
+      if VIDEO_MIME_TYPES.include?(file_content_type)
+        :video
+      elsif AUDIO_MIME_TYPES.include?(file_content_type)
+        :audio
+      else
+        :image
+      end
+    end
   end
 
   def set_meta
@@ -245,7 +267,7 @@ class MediaAttachment < ApplicationRecord
       frame_rate: movie.frame_rate,
       duration: movie.duration,
       bitrate: movie.bitrate,
-    }
+    }.compact
   end
 
   def reset_parent_cache
diff --git a/app/serializers/initial_state_serializer.rb b/app/serializers/initial_state_serializer.rb
index 184ed867c..3d825410e 100644
--- a/app/serializers/initial_state_serializer.rb
+++ b/app/serializers/initial_state_serializer.rb
@@ -60,7 +60,7 @@ class InitialStateSerializer < ActiveModel::Serializer
   end
 
   def media_attachments
-    { accept_content_types: MediaAttachment::IMAGE_FILE_EXTENSIONS + MediaAttachment::VIDEO_FILE_EXTENSIONS + MediaAttachment::IMAGE_MIME_TYPES + MediaAttachment::VIDEO_MIME_TYPES }
+    { accept_content_types: MediaAttachment::IMAGE_FILE_EXTENSIONS + MediaAttachment::VIDEO_FILE_EXTENSIONS + MediaAttachment::AUDIO_FILE_EXTENSIONS + MediaAttachment::IMAGE_MIME_TYPES + MediaAttachment::VIDEO_MIME_TYPES + MediaAttachment::AUDIO_MIME_TYPES }
   end
 
   private
diff --git a/app/services/post_status_service.rb b/app/services/post_status_service.rb
index b470a40e9..7830aee11 100644
--- a/app/services/post_status_service.rb
+++ b/app/services/post_status_service.rb
@@ -100,7 +100,7 @@ class PostStatusService < BaseService
 
     @media = @account.media_attachments.where(status_id: nil).where(id: @options[:media_ids].take(4).map(&:to_i))
 
-    raise Mastodon::ValidationError, I18n.t('media_attachments.validations.images_and_video') if @media.size > 1 && @media.find(&:video?)
+    raise Mastodon::ValidationError, I18n.t('media_attachments.validations.images_and_video') if @media.size > 1 && @media.find(&:audio_or_video?)
   end
 
   def language_from_option(str)
diff --git a/app/views/stream_entries/_detailed_status.html.haml b/app/views/stream_entries/_detailed_status.html.haml
index 95b96feef..069d0053f 100644
--- a/app/views/stream_entries/_detailed_status.html.haml
+++ b/app/views/stream_entries/_detailed_status.html.haml
@@ -27,7 +27,7 @@
           = render partial: 'stream_entries/poll', locals: { status: status, poll: status.preloadable_poll, autoplay: autoplay }
 
   - if !status.media_attachments.empty?
-    - if status.media_attachments.first.video?
+    - if status.media_attachments.first.audio_or_video?
       - video = status.media_attachments.first
       = react_component :video, src: video.file.url(:original), preview: video.file.url(:small), blurhash: video.blurhash, sensitive: !current_account&.user&.show_all_media? && status.sensitive? || current_account&.user&.hide_all_media?, width: 670, height: 380, detailed: true, inline: true, alt: video.description do
         = render partial: 'stream_entries/attachment_list', locals: { attachments: status.media_attachments }
diff --git a/app/views/stream_entries/_simple_status.html.haml b/app/views/stream_entries/_simple_status.html.haml
index 0f9c65af0..0b924f72f 100644
--- a/app/views/stream_entries/_simple_status.html.haml
+++ b/app/views/stream_entries/_simple_status.html.haml
@@ -31,7 +31,7 @@
           = render partial: 'stream_entries/poll', locals: { status: status, poll: status.preloadable_poll, autoplay: autoplay }
 
   - if !status.media_attachments.empty?
-    - if status.media_attachments.first.video?
+    - if status.media_attachments.first.audio_or_video?
       - video = status.media_attachments.first
       = react_component :video, src: video.file.url(:original), preview: video.file.url(:small), blurhash: video.blurhash, sensitive: !current_account&.user&.show_all_media? && status.sensitive? || current_account&.user&.hide_all_media?, width: 610, height: 343, inline: true, alt: video.description do
         = render partial: 'stream_entries/attachment_list', locals: { attachments: status.media_attachments }
-- 
cgit 


From 7696f77245c2302787d239da50248385b3292a5e Mon Sep 17 00:00:00 2001
From: Eugen Rochko <eugen@zeonfederated.com>
Date: Thu, 20 Jun 2019 02:52:34 +0200
Subject: Add moderation API (#9387)

Fix #8580
Fix #7143
---
 app/controllers/admin/accounts_controller.rb       |   1 +
 .../api/v1/admin/account_actions_controller.rb     |  32 +++++
 .../api/v1/admin/accounts_controller.rb            | 128 ++++++++++++++++++
 app/controllers/api/v1/admin/reports_controller.rb | 108 +++++++++++++++
 app/models/account.rb                              |   2 +
 app/models/account_filter.rb                       |   2 +
 app/models/concerns/user_roles.rb                  |  14 ++
 app/models/report.rb                               |   3 +
 app/models/report_filter.rb                        |   2 +
 app/models/user.rb                                 |   1 +
 app/serializers/rest/admin/account_serializer.rb   |  77 +++++++++++
 app/serializers/rest/admin/report_serializer.rb    |  16 +++
 config/initializers/doorkeeper.rb                  |   8 +-
 config/locales/doorkeeper.en.yml                   |   6 +
 config/routes.rb                                   |  23 ++++
 .../v1/admin/account_actions_controller_spec.rb    |  57 ++++++++
 .../api/v1/admin/accounts_controller_spec.rb       | 147 +++++++++++++++++++++
 .../api/v1/admin/reports_controller_spec.rb        | 109 +++++++++++++++
 18 files changed, 735 insertions(+), 1 deletion(-)
 create mode 100644 app/controllers/api/v1/admin/account_actions_controller.rb
 create mode 100644 app/controllers/api/v1/admin/accounts_controller.rb
 create mode 100644 app/controllers/api/v1/admin/reports_controller.rb
 create mode 100644 app/serializers/rest/admin/account_serializer.rb
 create mode 100644 app/serializers/rest/admin/report_serializer.rb
 create mode 100644 spec/controllers/api/v1/admin/account_actions_controller_spec.rb
 create mode 100644 spec/controllers/api/v1/admin/accounts_controller_spec.rb
 create mode 100644 spec/controllers/api/v1/admin/reports_controller_spec.rb

(limited to 'app/models')

diff --git a/app/controllers/admin/accounts_controller.rb b/app/controllers/admin/accounts_controller.rb
index b0d45ce47..0c7760d77 100644
--- a/app/controllers/admin/accounts_controller.rb
+++ b/app/controllers/admin/accounts_controller.rb
@@ -127,6 +127,7 @@ module Admin
         :by_domain,
         :active,
         :pending,
+        :disabled,
         :silenced,
         :suspended,
         :username,
diff --git a/app/controllers/api/v1/admin/account_actions_controller.rb b/app/controllers/api/v1/admin/account_actions_controller.rb
new file mode 100644
index 000000000..29c9b7107
--- /dev/null
+++ b/app/controllers/api/v1/admin/account_actions_controller.rb
@@ -0,0 +1,32 @@
+# frozen_string_literal: true
+
+class Api::V1::Admin::AccountActionsController < Api::BaseController
+  before_action -> { doorkeeper_authorize! :'admin:write', :'admin:write:accounts' }
+  before_action :require_staff!
+  before_action :set_account
+
+  def create
+    account_action                 = Admin::AccountAction.new(resource_params)
+    account_action.target_account  = @account
+    account_action.current_account = current_account
+    account_action.save!
+
+    render_empty
+  end
+
+  private
+
+  def set_account
+    @account = Account.find(params[:account_id])
+  end
+
+  def resource_params
+    params.permit(
+      :type,
+      :report_id,
+      :warning_preset_id,
+      :text,
+      :send_email_notification
+    )
+  end
+end
diff --git a/app/controllers/api/v1/admin/accounts_controller.rb b/app/controllers/api/v1/admin/accounts_controller.rb
new file mode 100644
index 000000000..c306180ca
--- /dev/null
+++ b/app/controllers/api/v1/admin/accounts_controller.rb
@@ -0,0 +1,128 @@
+# frozen_string_literal: true
+
+class Api::V1::Admin::AccountsController < Api::BaseController
+  include Authorization
+  include AccountableConcern
+
+  LIMIT = 100
+
+  before_action -> { doorkeeper_authorize! :'admin:read', :'admin:read:accounts' }, only: [:index, :show]
+  before_action -> { doorkeeper_authorize! :'admin:write', :'admin:write:accounts' }, except: [:index, :show]
+  before_action :require_staff!
+  before_action :set_accounts, only: :index
+  before_action :set_account, except: :index
+  before_action :require_local_account!, only: [:enable, :approve, :reject]
+
+  after_action :insert_pagination_headers, only: :index
+
+  FILTER_PARAMS = %i(
+    local
+    remote
+    by_domain
+    active
+    pending
+    disabled
+    silenced
+    suspended
+    username
+    display_name
+    email
+    ip
+    staff
+  ).freeze
+
+  PAGINATION_PARAMS = (%i(limit) + FILTER_PARAMS).freeze
+
+  def index
+    authorize :account, :index?
+    render json: @accounts, each_serializer: REST::Admin::AccountSerializer
+  end
+
+  def show
+    authorize @account, :show?
+    render json: @account, serializer: REST::Admin::AccountSerializer
+  end
+
+  def enable
+    authorize @account.user, :enable?
+    @account.user.enable!
+    log_action :enable, @account.user
+    render json: @account, serializer: REST::Admin::AccountSerializer
+  end
+
+  def approve
+    authorize @account.user, :approve?
+    @account.user.approve!
+    render json: @account, serializer: REST::Admin::AccountSerializer
+  end
+
+  def reject
+    authorize @account.user, :reject?
+    SuspendAccountService.new.call(@account, including_user: true, destroy: true, skip_distribution: true)
+    render json: @account, serializer: REST::Admin::AccountSerializer
+  end
+
+  def unsilence
+    authorize @account, :unsilence?
+    @account.unsilence!
+    log_action :unsilence, @account
+    render json: @account, serializer: REST::Admin::AccountSerializer
+  end
+
+  def unsuspend
+    authorize @account, :unsuspend?
+    @account.unsuspend!
+    log_action :unsuspend, @account
+    render json: @account, serializer: REST::Admin::AccountSerializer
+  end
+
+  private
+
+  def set_accounts
+    @accounts = filtered_accounts.order(id: :desc).includes(user: [:invite_request, :invite]).paginate_by_id(limit_param(LIMIT), params_slice(:max_id, :since_id, :min_id))
+  end
+
+  def set_account
+    @account = Account.find(params[:id])
+  end
+
+  def filtered_accounts
+    AccountFilter.new(filter_params).results
+  end
+
+  def filter_params
+    params.permit(*FILTER_PARAMS)
+  end
+
+  def insert_pagination_headers
+    set_pagination_headers(next_path, prev_path)
+  end
+
+  def next_path
+    api_v1_admin_accounts_url(pagination_params(max_id: pagination_max_id)) if records_continue?
+  end
+
+  def prev_path
+    api_v1_admin_accounts_url(pagination_params(min_id: pagination_since_id)) unless @accounts.empty?
+  end
+
+  def pagination_max_id
+    @accounts.last.id
+  end
+
+  def pagination_since_id
+    @accounts.first.id
+  end
+
+  def records_continue?
+    @accounts.size == limit_param(LIMIT)
+  end
+
+  def pagination_params(core_params)
+    params.slice(*PAGINATION_PARAMS).permit(*PAGINATION_PARAMS).merge(core_params)
+  end
+
+  def require_local_account!
+    forbidden unless @account.local? && @account.user.present?
+  end
+end
diff --git a/app/controllers/api/v1/admin/reports_controller.rb b/app/controllers/api/v1/admin/reports_controller.rb
new file mode 100644
index 000000000..1d48d3160
--- /dev/null
+++ b/app/controllers/api/v1/admin/reports_controller.rb
@@ -0,0 +1,108 @@
+# frozen_string_literal: true
+
+class Api::V1::Admin::ReportsController < Api::BaseController
+  include Authorization
+  include AccountableConcern
+
+  LIMIT = 100
+
+  before_action -> { doorkeeper_authorize! :'admin:read', :'admin:read:reports' }, only: [:index, :show]
+  before_action -> { doorkeeper_authorize! :'admin:write', :'admin:write:reports' }, except: [:index, :show]
+  before_action :require_staff!
+  before_action :set_reports, only: :index
+  before_action :set_report, except: :index
+
+  after_action :insert_pagination_headers, only: :index
+
+  FILTER_PARAMS = %i(
+    resolved
+    account_id
+    target_account_id
+  ).freeze
+
+  PAGINATION_PARAMS = (%i(limit) + FILTER_PARAMS).freeze
+
+  def index
+    authorize :report, :index?
+    render json: @reports, each_serializer: REST::Admin::ReportSerializer
+  end
+
+  def show
+    authorize @report, :show?
+    render json: @report, serializer: REST::Admin::ReportSerializer
+  end
+
+  def assign_to_self
+    authorize @report, :update?
+    @report.update!(assigned_account_id: current_account.id)
+    log_action :assigned_to_self, @report
+    render json: @report, serializer: REST::Admin::ReportSerializer
+  end
+
+  def unassign
+    authorize @report, :update?
+    @report.update!(assigned_account_id: nil)
+    log_action :unassigned, @report
+    render json: @report, serializer: REST::Admin::ReportSerializer
+  end
+
+  def reopen
+    authorize @report, :update?
+    @report.unresolve!
+    log_action :reopen, @report
+    render json: @report, serializer: REST::Admin::ReportSerializer
+  end
+
+  def resolve
+    authorize @report, :update?
+    @report.resolve!(current_account)
+    log_action :resolve, @report
+    render json: @report, serializer: REST::Admin::ReportSerializer
+  end
+
+  private
+
+  def set_reports
+    @reports = filtered_reports.order(id: :desc).with_accounts.paginate_by_id(limit_param(LIMIT), params_slice(:max_id, :since_id, :min_id))
+  end
+
+  def set_report
+    @report = Report.find(params[:id])
+  end
+
+  def filtered_reports
+    ReportFilter.new(filter_params).results
+  end
+
+  def filter_params
+    params.permit(*FILTER_PARAMS)
+  end
+
+  def insert_pagination_headers
+    set_pagination_headers(next_path, prev_path)
+  end
+
+  def next_path
+    api_v1_admin_reports_url(pagination_params(max_id: pagination_max_id)) if records_continue?
+  end
+
+  def prev_path
+    api_v1_admin_reports_url(pagination_params(min_id: pagination_since_id)) unless @reports.empty?
+  end
+
+  def pagination_max_id
+    @reports.last.id
+  end
+
+  def pagination_since_id
+    @reports.first.id
+  end
+
+  def records_continue?
+    @reports.size == limit_param(LIMIT)
+  end
+
+  def pagination_params(core_params)
+    params.slice(*PAGINATION_PARAMS).permit(*PAGINATION_PARAMS).merge(core_params)
+  end
+end
diff --git a/app/models/account.rb b/app/models/account.rb
index c977f887c..9276aa927 100644
--- a/app/models/account.rb
+++ b/app/models/account.rb
@@ -106,6 +106,8 @@ class Account < ApplicationRecord
            :confirmed?,
            :approved?,
            :pending?,
+           :disabled?,
+           :role,
            :admin?,
            :moderator?,
            :staff?,
diff --git a/app/models/account_filter.rb b/app/models/account_filter.rb
index d2503100c..c3b1fe08d 100644
--- a/app/models/account_filter.rb
+++ b/app/models/account_filter.rb
@@ -37,6 +37,8 @@ class AccountFilter
       Account.without_suspended
     when 'pending'
       accounts_with_users.merge User.pending
+    when 'disabled'
+      accounts_with_users.merge User.disabled
     when 'silenced'
       Account.silenced
     when 'suspended'
diff --git a/app/models/concerns/user_roles.rb b/app/models/concerns/user_roles.rb
index 58dffdc46..a42b4a172 100644
--- a/app/models/concerns/user_roles.rb
+++ b/app/models/concerns/user_roles.rb
@@ -13,6 +13,20 @@ module UserRoles
     admin? || moderator?
   end
 
+  def role=(value)
+    case value
+    when 'admin'
+      self.admin     = true
+      self.moderator = false
+    when 'moderator'
+      self.admin     = false
+      self.moderator = true
+    else
+      self.admin     = false
+      self.moderator = false
+    end
+  end
+
   def role
     if admin?
       'admin'
diff --git a/app/models/report.rb b/app/models/report.rb
index 86c303798..5192ceef7 100644
--- a/app/models/report.rb
+++ b/app/models/report.rb
@@ -17,6 +17,8 @@
 #
 
 class Report < ApplicationRecord
+  include Paginable
+
   belongs_to :account
   belongs_to :target_account, class_name: 'Account'
   belongs_to :action_taken_by_account, class_name: 'Account', optional: true
@@ -26,6 +28,7 @@ class Report < ApplicationRecord
 
   scope :unresolved, -> { where(action_taken: false) }
   scope :resolved,   -> { where(action_taken: true) }
+  scope :with_accounts, -> { includes([:account, :target_account, :action_taken_by_account, :assigned_account].each_with_object({}) { |k, h| h[k] = { user: [:invite_request, :invite] } }) }
 
   validates :comment, length: { maximum: 1000 }
 
diff --git a/app/models/report_filter.rb b/app/models/report_filter.rb
index 56ab28df7..a392d60c3 100644
--- a/app/models/report_filter.rb
+++ b/app/models/report_filter.rb
@@ -9,9 +9,11 @@ class ReportFilter
 
   def results
     scope = Report.unresolved
+
     params.each do |key, value|
       scope = scope.merge scope_for(key, value)
     end
+
     scope
   end
 
diff --git a/app/models/user.rb b/app/models/user.rb
index 4abf124fc..50873dd01 100644
--- a/app/models/user.rb
+++ b/app/models/user.rb
@@ -87,6 +87,7 @@ class User < ApplicationRecord
   scope :approved, -> { where(approved: true) }
   scope :confirmed, -> { where.not(confirmed_at: nil) }
   scope :enabled, -> { where(disabled: false) }
+  scope :disabled, -> { where(disabled: true) }
   scope :inactive, -> { where(arel_table[:current_sign_in_at].lt(ACTIVE_DURATION.ago)) }
   scope :active, -> { confirmed.where(arel_table[:current_sign_in_at].gteq(ACTIVE_DURATION.ago)).joins(:account).where(accounts: { suspended_at: nil }) }
   scope :matches_email, ->(value) { where(arel_table[:email].matches("#{value}%")) }
diff --git a/app/serializers/rest/admin/account_serializer.rb b/app/serializers/rest/admin/account_serializer.rb
new file mode 100644
index 000000000..f579d3302
--- /dev/null
+++ b/app/serializers/rest/admin/account_serializer.rb
@@ -0,0 +1,77 @@
+# frozen_string_literal: true
+
+class REST::Admin::AccountSerializer < ActiveModel::Serializer
+  attributes :id, :username, :domain, :created_at,
+             :email, :ip, :role, :confirmed, :suspended,
+             :silenced, :disabled, :approved, :locale,
+             :invite_request
+
+  attribute :created_by_application_id, if: :created_by_application?
+  attribute :invited_by_account_id, if: :invited?
+
+  has_one :account, serializer: REST::AccountSerializer
+
+  def id
+    object.id.to_s
+  end
+
+  def email
+    object.user_email
+  end
+
+  def ip
+    object.user_current_sign_in_ip.to_s.presence
+  end
+
+  def role
+    object.user_role
+  end
+
+  def suspended
+    object.suspended?
+  end
+
+  def silenced
+    object.silenced?
+  end
+
+  def confirmed
+    object.user_confirmed?
+  end
+
+  def disabled
+    object.user_disabled?
+  end
+
+  def approved
+    object.user_approved?
+  end
+
+  def account
+    object
+  end
+
+  def locale
+    object.user_locale
+  end
+
+  def created_by_application_id
+    object.user&.created_by_application_id&.to_s&.presence
+  end
+
+  def invite_request
+    object.user&.invite_request&.text
+  end
+
+  def invited_by_account_id
+    object.user&.invite&.user&.account_id&.to_s&.presence
+  end
+
+  def invited?
+    object.user&.invited?
+  end
+
+  def created_by_application?
+    object.user&.created_by_application_id&.present?
+  end
+end
diff --git a/app/serializers/rest/admin/report_serializer.rb b/app/serializers/rest/admin/report_serializer.rb
new file mode 100644
index 000000000..7a77132c0
--- /dev/null
+++ b/app/serializers/rest/admin/report_serializer.rb
@@ -0,0 +1,16 @@
+# frozen_string_literal: true
+
+class REST::Admin::ReportSerializer < ActiveModel::Serializer
+  attributes :id, :action_taken, :comment, :created_at, :updated_at
+
+  has_one :account, serializer: REST::Admin::AccountSerializer
+  has_one :target_account, serializer: REST::Admin::AccountSerializer
+  has_one :assigned_account, serializer: REST::Admin::AccountSerializer
+  has_one :action_taken_by_account, serializer: REST::Admin::AccountSerializer
+
+  has_many :statuses, serializer: REST::StatusSerializer
+
+  def id
+    object.id.to_s
+  end
+end
diff --git a/config/initializers/doorkeeper.rb b/config/initializers/doorkeeper.rb
index 367eead6a..914b3c001 100644
--- a/config/initializers/doorkeeper.rb
+++ b/config/initializers/doorkeeper.rb
@@ -80,7 +80,13 @@ Doorkeeper.configure do
                   :'read:search',
                   :'read:statuses',
                   :follow,
-                  :push
+                  :push,
+                  :'admin:read',
+                  :'admin:read:accounts',
+                  :'admin:read:reports',
+                  :'admin:write',
+                  :'admin:write:accounts',
+                  :'admin:write:reports'
 
   # Change the way client credentials are retrieved from the request object.
   # By default it retrieves first from the `HTTP_AUTHORIZATION` header, then
diff --git a/config/locales/doorkeeper.en.yml b/config/locales/doorkeeper.en.yml
index f1fe03716..d9b7c2c8e 100644
--- a/config/locales/doorkeeper.en.yml
+++ b/config/locales/doorkeeper.en.yml
@@ -114,6 +114,12 @@ en:
       application:
         title: OAuth authorization required
     scopes:
+      admin:read: read all data on the server
+      admin:read:accounts: read sensitive information of all accounts
+      admin:read:reports: read sensitive information of all reports and reported accounts
+      admin:write: modify all data on the server
+      admin:write:accounts: perform moderation actions on accounts
+      admin:write:reports: perform moderation actions on reports
       follow: modify account relationships
       push: receive your push notifications
       read: read all your account's data
diff --git a/config/routes.rb b/config/routes.rb
index 145079c69..764db8db2 100644
--- a/config/routes.rb
+++ b/config/routes.rb
@@ -398,6 +398,29 @@ Rails.application.routes.draw do
       namespace :push do
         resource :subscription, only: [:create, :show, :update, :destroy]
       end
+
+      namespace :admin do
+        resources :accounts, only: [:index, :show] do
+          member do
+            post :enable
+            post :unsilence
+            post :unsuspend
+            post :approve
+            post :reject
+          end
+
+          resource :action, only: [:create], controller: 'account_actions'
+        end
+
+        resources :reports, only: [:index, :show] do
+          member do
+            post :assign_to_self
+            post :unassign
+            post :reopen
+            post :resolve
+          end
+        end
+      end
     end
 
     namespace :v2 do
diff --git a/spec/controllers/api/v1/admin/account_actions_controller_spec.rb b/spec/controllers/api/v1/admin/account_actions_controller_spec.rb
new file mode 100644
index 000000000..a5a8f4bb0
--- /dev/null
+++ b/spec/controllers/api/v1/admin/account_actions_controller_spec.rb
@@ -0,0 +1,57 @@
+require 'rails_helper'
+
+RSpec.describe Api::V1::Admin::AccountActionsController, type: :controller do
+  render_views
+
+  let(:role)   { 'moderator' }
+  let(:user)   { Fabricate(:user, role: role, account: Fabricate(:account, username: 'alice')) }
+  let(:scopes) { 'admin:read admin:write' }
+  let(:token)  { Fabricate(:accessible_access_token, resource_owner_id: user.id, scopes: scopes) }
+  let(:account) { Fabricate(:user).account }
+
+  before do
+    allow(controller).to receive(:doorkeeper_token) { token }
+  end
+
+  shared_examples 'forbidden for wrong scope' do |wrong_scope|
+    let(:scopes) { wrong_scope }
+
+    it 'returns http forbidden' do
+      expect(response).to have_http_status(403)
+    end
+  end
+
+  shared_examples 'forbidden for wrong role' do |wrong_role|
+    let(:role) { wrong_role }
+
+    it 'returns http forbidden' do
+      expect(response).to have_http_status(403)
+    end
+  end
+
+  describe 'POST #create' do
+    before do
+      post :create, params: { account_id: account.id, type: 'disable' }
+    end
+
+    it_behaves_like 'forbidden for wrong scope', 'write:statuses'
+    it_behaves_like 'forbidden for wrong role', 'user'
+
+    it 'returns http success' do
+      expect(response).to have_http_status(200)
+    end
+
+    it 'performs action against account' do
+      expect(account.reload.user_disabled?).to be true
+    end
+
+    it 'logs action' do
+      log_item = Admin::ActionLog.last
+
+      expect(log_item).to_not be_nil
+      expect(log_item.action).to eq :disable
+      expect(log_item.account_id).to eq user.account_id
+      expect(log_item.target_id).to eq account.user.id
+    end
+  end
+end
diff --git a/spec/controllers/api/v1/admin/accounts_controller_spec.rb b/spec/controllers/api/v1/admin/accounts_controller_spec.rb
new file mode 100644
index 000000000..f3f9946ba
--- /dev/null
+++ b/spec/controllers/api/v1/admin/accounts_controller_spec.rb
@@ -0,0 +1,147 @@
+require 'rails_helper'
+
+RSpec.describe Api::V1::Admin::AccountsController, type: :controller do
+  render_views
+
+  let(:role)   { 'moderator' }
+  let(:user)   { Fabricate(:user, role: role, account: Fabricate(:account, username: 'alice')) }
+  let(:scopes) { 'admin:read admin:write' }
+  let(:token)  { Fabricate(:accessible_access_token, resource_owner_id: user.id, scopes: scopes) }
+  let(:account) { Fabricate(:user).account }
+
+  before do
+    allow(controller).to receive(:doorkeeper_token) { token }
+  end
+
+  shared_examples 'forbidden for wrong scope' do |wrong_scope|
+    let(:scopes) { wrong_scope }
+
+    it 'returns http forbidden' do
+      expect(response).to have_http_status(403)
+    end
+  end
+
+  shared_examples 'forbidden for wrong role' do |wrong_role|
+    let(:role) { wrong_role }
+
+    it 'returns http forbidden' do
+      expect(response).to have_http_status(403)
+    end
+  end
+
+  describe 'GET #index' do
+    before do
+      get :index
+    end
+
+    it_behaves_like 'forbidden for wrong scope', 'write:statuses'
+    it_behaves_like 'forbidden for wrong role', 'user'
+
+    it 'returns http success' do
+      expect(response).to have_http_status(200)
+    end
+  end
+
+  describe 'GET #show' do
+    before do
+      get :show, params: { id: account.id }
+    end
+
+    it_behaves_like 'forbidden for wrong scope', 'write:statuses'
+    it_behaves_like 'forbidden for wrong role', 'user'
+
+    it 'returns http success' do
+      expect(response).to have_http_status(200)
+    end
+  end
+
+  describe 'POST #approve' do
+    before do
+      account.user.update(approved: false)
+      post :approve, params: { id: account.id }
+    end
+
+    it_behaves_like 'forbidden for wrong scope', 'write:statuses'
+    it_behaves_like 'forbidden for wrong role', 'user'
+
+    it 'returns http success' do
+      expect(response).to have_http_status(200)
+    end
+
+    it 'approves user' do
+      expect(account.reload.user_approved?).to be true
+    end
+  end
+
+  describe 'POST #reject' do
+    before do
+      account.user.update(approved: false)
+      post :reject, params: { id: account.id }
+    end
+
+    it_behaves_like 'forbidden for wrong scope', 'write:statuses'
+    it_behaves_like 'forbidden for wrong role', 'user'
+
+    it 'returns http success' do
+      expect(response).to have_http_status(200)
+    end
+
+    it 'removes user' do
+      expect(User.where(id: account.user.id).count).to eq 0
+    end
+  end
+
+  describe 'POST #enable' do
+    before do
+      account.user.update(disabled: true)
+      post :enable, params: { id: account.id }
+    end
+
+    it_behaves_like 'forbidden for wrong scope', 'write:statuses'
+    it_behaves_like 'forbidden for wrong role', 'user'
+
+    it 'returns http success' do
+      expect(response).to have_http_status(200)
+    end
+
+    it 'enables user' do
+      expect(account.reload.user_disabled?).to be false
+    end
+  end
+
+  describe 'POST #unsuspend' do
+    before do
+      account.touch(:suspended_at)
+      post :unsuspend, params: { id: account.id }
+    end
+
+    it_behaves_like 'forbidden for wrong scope', 'write:statuses'
+    it_behaves_like 'forbidden for wrong role', 'user'
+
+    it 'returns http success' do
+      expect(response).to have_http_status(200)
+    end
+
+    it 'unsuspends account' do
+      expect(account.reload.suspended?).to be false
+    end
+  end
+
+  describe 'POST #unsilence' do
+    before do
+      account.touch(:silenced_at)
+      post :unsilence, params: { id: account.id }
+    end
+
+    it_behaves_like 'forbidden for wrong scope', 'write:statuses'
+    it_behaves_like 'forbidden for wrong role', 'user'
+
+    it 'returns http success' do
+      expect(response).to have_http_status(200)
+    end
+
+    it 'unsilences account' do
+      expect(account.reload.silenced?).to be false
+    end
+  end
+end
diff --git a/spec/controllers/api/v1/admin/reports_controller_spec.rb b/spec/controllers/api/v1/admin/reports_controller_spec.rb
new file mode 100644
index 000000000..4ed3c5dc4
--- /dev/null
+++ b/spec/controllers/api/v1/admin/reports_controller_spec.rb
@@ -0,0 +1,109 @@
+require 'rails_helper'
+
+RSpec.describe Api::V1::Admin::ReportsController, type: :controller do
+  render_views
+
+  let(:role)   { 'moderator' }
+  let(:user)   { Fabricate(:user, role: role, account: Fabricate(:account, username: 'alice')) }
+  let(:scopes) { 'admin:read admin:write' }
+  let(:token)  { Fabricate(:accessible_access_token, resource_owner_id: user.id, scopes: scopes) }
+  let(:report) { Fabricate(:report) }
+
+  before do
+    allow(controller).to receive(:doorkeeper_token) { token }
+  end
+
+  shared_examples 'forbidden for wrong scope' do |wrong_scope|
+    let(:scopes) { wrong_scope }
+
+    it 'returns http forbidden' do
+      expect(response).to have_http_status(403)
+    end
+  end
+
+  shared_examples 'forbidden for wrong role' do |wrong_role|
+    let(:role) { wrong_role }
+
+    it 'returns http forbidden' do
+      expect(response).to have_http_status(403)
+    end
+  end
+
+  describe 'GET #index' do
+    before do
+      get :index
+    end
+
+    it_behaves_like 'forbidden for wrong scope', 'write:statuses'
+    it_behaves_like 'forbidden for wrong role', 'user'
+
+    it 'returns http success' do
+      expect(response).to have_http_status(200)
+    end
+  end
+
+  describe 'GET #show' do
+    before do
+      get :show, params: { id: report.id }
+    end
+
+    it_behaves_like 'forbidden for wrong scope', 'write:statuses'
+    it_behaves_like 'forbidden for wrong role', 'user'
+
+    it 'returns http success' do
+      expect(response).to have_http_status(200)
+    end
+  end
+
+  describe 'POST #resolve' do
+    before do
+      post :resolve, params: { id: report.id }
+    end
+
+    it_behaves_like 'forbidden for wrong scope', 'write:statuses'
+    it_behaves_like 'forbidden for wrong role', 'user'
+
+    it 'returns http success' do
+      expect(response).to have_http_status(200)
+    end
+  end
+
+  describe 'POST #reopen' do
+    before do
+      post :reopen, params: { id: report.id }
+    end
+
+    it_behaves_like 'forbidden for wrong scope', 'write:statuses'
+    it_behaves_like 'forbidden for wrong role', 'user'
+
+    it 'returns http success' do
+      expect(response).to have_http_status(200)
+    end
+  end
+
+  describe 'POST #assign_to_self' do
+    before do
+      post :assign_to_self, params: { id: report.id }
+    end
+
+    it_behaves_like 'forbidden for wrong scope', 'write:statuses'
+    it_behaves_like 'forbidden for wrong role', 'user'
+
+    it 'returns http success' do
+      expect(response).to have_http_status(200)
+    end
+  end
+
+  describe 'POST #unassign' do
+    before do
+      post :unassign, params: { id: report.id }
+    end
+
+    it_behaves_like 'forbidden for wrong scope', 'write:statuses'
+    it_behaves_like 'forbidden for wrong role', 'user'
+
+    it 'returns http success' do
+      expect(response).to have_http_status(200)
+    end
+  end
+end
-- 
cgit 


From 8f23726918fd8ded18ce9e55e6199df87abcc7cf Mon Sep 17 00:00:00 2001
From: Eugen Rochko <eugen@zeonfederated.com>
Date: Thu, 20 Jun 2019 10:52:36 +0200
Subject: Fix converted media being saved with original extension and mime type
 (#11130)

---
 app/models/concerns/attachmentable.rb |  2 +-
 app/models/media_attachment.rb        |  8 +++++---
 config/application.rb                 |  1 +
 lib/paperclip/type_corrector.rb       | 19 +++++++++++++++++++
 4 files changed, 26 insertions(+), 4 deletions(-)
 create mode 100644 lib/paperclip/type_corrector.rb

(limited to 'app/models')

diff --git a/app/models/concerns/attachmentable.rb b/app/models/concerns/attachmentable.rb
index de4cf8775..f4e37f1e6 100644
--- a/app/models/concerns/attachmentable.rb
+++ b/app/models/concerns/attachmentable.rb
@@ -1,6 +1,6 @@
 # frozen_string_literal: true
 
-require 'mime/types'
+require 'mime/types/columnar'
 
 module Attachmentable
   extend ActiveSupport::Concern
diff --git a/app/models/media_attachment.rb b/app/models/media_attachment.rb
index 5ca218411..ae94ce68a 100644
--- a/app/models/media_attachment.rb
+++ b/app/models/media_attachment.rb
@@ -70,12 +70,14 @@ class MediaAttachment < ApplicationRecord
   AUDIO_STYLES = {
     original: {
       format: 'ogg',
+      content_type: 'audio/ogg',
       convert_options: {},
     },
   }.freeze
 
   VIDEO_FORMAT = {
     format: 'mp4',
+    content_type: 'video/mp4',
     convert_options: {
       output: {
         'loglevel' => 'fatal',
@@ -189,11 +191,11 @@ class MediaAttachment < ApplicationRecord
       if f.file_content_type == 'image/gif'
         [:gif_transcoder, :blurhash_transcoder]
       elsif VIDEO_MIME_TYPES.include?(f.file_content_type)
-        [:video_transcoder, :blurhash_transcoder]
+        [:video_transcoder, :blurhash_transcoder, :type_corrector]
       elsif AUDIO_MIME_TYPES.include?(f.file_content_type)
-        [:transcoder]
+        [:transcoder, :type_corrector]
       else
-        [:lazy_thumbnail, :blurhash_transcoder]
+        [:lazy_thumbnail, :blurhash_transcoder, :type_corrector]
       end
     end
   end
diff --git a/config/application.rb b/config/application.rb
index 74006f5fa..4534ede49 100644
--- a/config/application.rb
+++ b/config/application.rb
@@ -10,6 +10,7 @@ require_relative '../app/lib/exceptions'
 require_relative '../lib/paperclip/lazy_thumbnail'
 require_relative '../lib/paperclip/gif_transcoder'
 require_relative '../lib/paperclip/video_transcoder'
+require_relative '../lib/paperclip/type_corrector'
 require_relative '../lib/mastodon/snowflake'
 require_relative '../lib/mastodon/version'
 require_relative '../lib/devise/ldap_authenticatable'
diff --git a/lib/paperclip/type_corrector.rb b/lib/paperclip/type_corrector.rb
new file mode 100644
index 000000000..0b0c10a56
--- /dev/null
+++ b/lib/paperclip/type_corrector.rb
@@ -0,0 +1,19 @@
+# frozen_string_literal: true
+
+require 'mime/types/columnar'
+
+module Paperclip
+  class TypeCorrector < Paperclip::Processor
+    def make
+      target_extension = options[:format]
+      extension        = File.extname(attachment.instance.file_file_name)
+
+      return @file unless options[:style] == :original && target_extension && extension != target_extension
+
+      attachment.instance.file_content_type = options[:content_type] || attachment.instance.file_content_type
+      attachment.instance.file_file_name    = File.basename(attachment.instance.file_file_name, '.*') + '.' + target_extension
+
+      @file
+    end
+  end
+end
-- 
cgit 


From 49ebda4d49af50b375126e4a8285686fb6448a60 Mon Sep 17 00:00:00 2001
From: Eugen Rochko <eugen@zeonfederated.com>
Date: Fri, 21 Jun 2019 22:59:44 +0200
Subject: Change audio format from ogg to mp3 for wider compatibility (#11141)

* Change audio format from ogg to mp3 for wider compatibility

* Add media description as title to links of unknown media attachments
---
 app/javascript/mastodon/components/media_gallery.js |  2 +-
 app/javascript/styles/mastodon/components.scss      |  1 +
 app/models/media_attachment.rb                      | 10 +++++++---
 3 files changed, 9 insertions(+), 4 deletions(-)

(limited to 'app/models')

diff --git a/app/javascript/mastodon/components/media_gallery.js b/app/javascript/mastodon/components/media_gallery.js
index 56618462b..77bac61ee 100644
--- a/app/javascript/mastodon/components/media_gallery.js
+++ b/app/javascript/mastodon/components/media_gallery.js
@@ -157,7 +157,7 @@ class Item extends React.PureComponent {
     if (attachment.get('type') === 'unknown') {
       return (
         <div className={classNames('media-gallery__item', { standalone })} key={attachment.get('id')} style={{ left: left, top: top, right: right, bottom: bottom, width: `${width}%`, height: `${height}%` }}>
-          <a className='media-gallery__item-thumbnail' href={attachment.get('remote_url')} target='_blank' style={{ cursor: 'pointer' }}>
+          <a className='media-gallery__item-thumbnail' href={attachment.get('remote_url')} target='_blank' style={{ cursor: 'pointer' }} title={attachment.get('description')}>
             <canvas width={32} height={32} ref={this.setCanvasRef} className='media-gallery__preview' />
           </a>
         </div>
diff --git a/app/javascript/styles/mastodon/components.scss b/app/javascript/styles/mastodon/components.scss
index 545427d22..4a0d64ad3 100644
--- a/app/javascript/styles/mastodon/components.scss
+++ b/app/javascript/styles/mastodon/components.scss
@@ -557,6 +557,7 @@
 
     .compose-form__upload-thumbnail {
       border-radius: 4px;
+      background-color: $base-shadow-color;
       background-position: center;
       background-size: cover;
       background-repeat: no-repeat;
diff --git a/app/models/media_attachment.rb b/app/models/media_attachment.rb
index ae94ce68a..4c55d4b98 100644
--- a/app/models/media_attachment.rb
+++ b/app/models/media_attachment.rb
@@ -69,9 +69,13 @@ class MediaAttachment < ApplicationRecord
 
   AUDIO_STYLES = {
     original: {
-      format: 'ogg',
-      content_type: 'audio/ogg',
-      convert_options: {},
+      format: 'mp3',
+      content_type: 'audio/mpeg',
+      convert_options: {
+        output: {
+          'q:a' => 2,
+        },
+      },
     },
   }.freeze
 
-- 
cgit 


From 707ddf7808f90e3ab042d7642d368c2ce8e95e6f Mon Sep 17 00:00:00 2001
From: Eugen Rochko <eugen@zeonfederated.com>
Date: Sat, 22 Jun 2019 00:13:10 +0200
Subject: Change domain blocks to automatically support subdomains (#11138)

* Change domain blocks to automatically support subdomains

If a more authoritative domain is blocked (example.com), then the
same block will be applied to a subdomain (foo.example.com)

* Match subdomains of existing accounts when blocking/unblocking domains

* Improve code style
---
 app/controllers/admin/domain_blocks_controller.rb  |  2 +-
 app/controllers/admin/instances_controller.rb      |  2 +-
 app/controllers/media_proxy_controller.rb          |  2 +-
 app/lib/activitypub/activity/create.rb             |  2 +-
 app/lib/activitypub/activity/flag.rb               |  2 +-
 app/lib/ostatus/activity/creation.rb               |  4 +--
 app/models/account.rb                              |  1 +
 app/models/custom_emoji.rb                         |  1 +
 app/models/domain_block.rb                         | 33 ++++++++++++++++++++--
 app/models/instance.rb                             |  2 +-
 .../activitypub/process_account_service.rb         |  2 +-
 app/services/block_domain_service.rb               |  4 +--
 app/services/resolve_account_service.rb            |  2 +-
 app/services/unblock_domain_service.rb             |  3 +-
 app/services/update_remote_profile_service.rb      |  4 +--
 spec/models/account_spec.rb                        | 17 +++++++++++
 spec/models/domain_block_spec.rb                   | 31 +++++++++++++++-----
 17 files changed, 89 insertions(+), 25 deletions(-)

(limited to 'app/models')

diff --git a/app/controllers/admin/domain_blocks_controller.rb b/app/controllers/admin/domain_blocks_controller.rb
index 71597763b..377cac8ad 100644
--- a/app/controllers/admin/domain_blocks_controller.rb
+++ b/app/controllers/admin/domain_blocks_controller.rb
@@ -13,7 +13,7 @@ module Admin
       authorize :domain_block, :create?
 
       @domain_block = DomainBlock.new(resource_params)
-      existing_domain_block = resource_params[:domain].present? ? DomainBlock.find_by(domain: resource_params[:domain]) : nil
+      existing_domain_block = resource_params[:domain].present? ? DomainBlock.rule_for(resource_params[:domain]) : nil
 
       if existing_domain_block.present? && !@domain_block.stricter_than?(existing_domain_block)
         @domain_block.save
diff --git a/app/controllers/admin/instances_controller.rb b/app/controllers/admin/instances_controller.rb
index 6dd659a30..7888e844f 100644
--- a/app/controllers/admin/instances_controller.rb
+++ b/app/controllers/admin/instances_controller.rb
@@ -18,7 +18,7 @@ module Admin
       @blocks_count    = Block.where(target_account: Account.where(domain: params[:id])).count
       @available       = DeliveryFailureTracker.available?(Account.select(:shared_inbox_url).where(domain: params[:id]).first&.shared_inbox_url)
       @media_storage   = MediaAttachment.where(account: Account.where(domain: params[:id])).sum(:file_file_size)
-      @domain_block    = DomainBlock.find_by(domain: params[:id])
+      @domain_block    = DomainBlock.rule_for(params[:id])
     end
 
     private
diff --git a/app/controllers/media_proxy_controller.rb b/app/controllers/media_proxy_controller.rb
index 950cf6d09..8fc18dd06 100644
--- a/app/controllers/media_proxy_controller.rb
+++ b/app/controllers/media_proxy_controller.rb
@@ -39,6 +39,6 @@ class MediaProxyController < ApplicationController
   end
 
   def reject_media?
-    DomainBlock.find_by(domain: @media_attachment.account.domain)&.reject_media?
+    DomainBlock.reject_media?(@media_attachment.account.domain)
   end
 end
diff --git a/app/lib/activitypub/activity/create.rb b/app/lib/activitypub/activity/create.rb
index f55dd35b2..487e8e91e 100644
--- a/app/lib/activitypub/activity/create.rb
+++ b/app/lib/activitypub/activity/create.rb
@@ -380,7 +380,7 @@ class ActivityPub::Activity::Create < ActivityPub::Activity
 
   def skip_download?
     return @skip_download if defined?(@skip_download)
-    @skip_download ||= DomainBlock.find_by(domain: @account.domain)&.reject_media?
+    @skip_download ||= DomainBlock.reject_media?(@account.domain)
   end
 
   def reply_to_local?
diff --git a/app/lib/activitypub/activity/flag.rb b/app/lib/activitypub/activity/flag.rb
index f73b93058..1659bc61f 100644
--- a/app/lib/activitypub/activity/flag.rb
+++ b/app/lib/activitypub/activity/flag.rb
@@ -23,7 +23,7 @@ class ActivityPub::Activity::Flag < ActivityPub::Activity
   private
 
   def skip_reports?
-    DomainBlock.find_by(domain: @account.domain)&.reject_reports?
+    DomainBlock.reject_reports?(@account.domain)
   end
 
   def object_uris
diff --git a/app/lib/ostatus/activity/creation.rb b/app/lib/ostatus/activity/creation.rb
index 3840c8fbf..60de712db 100644
--- a/app/lib/ostatus/activity/creation.rb
+++ b/app/lib/ostatus/activity/creation.rb
@@ -148,7 +148,7 @@ class OStatus::Activity::Creation < OStatus::Activity::Base
   end
 
   def save_media
-    do_not_download = DomainBlock.find_by(domain: @account.domain)&.reject_media?
+    do_not_download = DomainBlock.reject_media?(@account.domain)
     media_attachments = []
 
     @xml.xpath('./xmlns:link[@rel="enclosure"]', xmlns: OStatus::TagManager::XMLNS).each do |link|
@@ -176,7 +176,7 @@ class OStatus::Activity::Creation < OStatus::Activity::Base
   end
 
   def save_emojis(parent)
-    do_not_download = DomainBlock.find_by(domain: parent.account.domain)&.reject_media?
+    do_not_download = DomainBlock.reject_media?(parent.account.domain)
 
     return if do_not_download
 
diff --git a/app/models/account.rb b/app/models/account.rb
index 9276aa927..c588451fc 100644
--- a/app/models/account.rb
+++ b/app/models/account.rb
@@ -98,6 +98,7 @@ class Account < ApplicationRecord
   scope :tagged_with, ->(tag) { joins(:accounts_tags).where(accounts_tags: { tag_id: tag }) }
   scope :by_recent_status, -> { order(Arel.sql('(case when account_stats.last_status_at is null then 1 else 0 end) asc, account_stats.last_status_at desc')) }
   scope :popular, -> { order('account_stats.followers_count desc') }
+  scope :by_domain_and_subdomains, ->(domain) { where(domain: domain).or(where(arel_table[:domain].matches('%.' + domain))) }
 
   delegate :email,
            :unconfirmed_email,
diff --git a/app/models/custom_emoji.rb b/app/models/custom_emoji.rb
index d3cc70504..e73cd9bd2 100644
--- a/app/models/custom_emoji.rb
+++ b/app/models/custom_emoji.rb
@@ -39,6 +39,7 @@ class CustomEmoji < ApplicationRecord
   scope :local,      -> { where(domain: nil) }
   scope :remote,     -> { where.not(domain: nil) }
   scope :alphabetic, -> { order(domain: :asc, shortcode: :asc) }
+  scope :by_domain_and_subdomains, ->(domain) { where(domain: domain).or(where(arel_table[:domain].matches('%.' + domain))) }
 
   remotable_attachment :image, LIMIT
 
diff --git a/app/models/domain_block.rb b/app/models/domain_block.rb
index 84c08c158..25d3b87ef 100644
--- a/app/models/domain_block.rb
+++ b/app/models/domain_block.rb
@@ -24,14 +24,41 @@ class DomainBlock < ApplicationRecord
 
   scope :matches_domain, ->(value) { where(arel_table[:domain].matches("%#{value}%")) }
 
-  def self.blocked?(domain)
-    where(domain: domain, severity: :suspend).exists?
+  class << self
+    def suspend?(domain)
+      !!rule_for(domain)&.suspend?
+    end
+
+    def silence?(domain)
+      !!rule_for(domain)&.silence?
+    end
+
+    def reject_media?(domain)
+      !!rule_for(domain)&.reject_media?
+    end
+
+    def reject_reports?(domain)
+      !!rule_for(domain)&.reject_reports?
+    end
+
+    alias blocked? suspend?
+
+    def rule_for(domain)
+      return if domain.blank?
+
+      uri      = Addressable::URI.new.tap { |u| u.host = domain.gsub(/[\/]/, '') }
+      segments = uri.normalized_host.split('.')
+      variants = segments.map.with_index { |_, i| segments[i..-1].join('.') }
+
+      where(domain: variants[0..-2]).order(Arel.sql('char_length(domain) desc')).first
+    end
   end
 
   def stricter_than?(other_block)
-    return true if suspend?
+    return true  if suspend?
     return false if other_block.suspend? && (silence? || noop?)
     return false if other_block.silence? && noop?
+
     (reject_media || !other_block.reject_media) && (reject_reports || !other_block.reject_reports)
   end
 
diff --git a/app/models/instance.rb b/app/models/instance.rb
index 7bf000d40..a01db1212 100644
--- a/app/models/instance.rb
+++ b/app/models/instance.rb
@@ -8,7 +8,7 @@ class Instance
   def initialize(resource)
     @domain         = resource.domain
     @accounts_count = resource.is_a?(DomainBlock) ? nil : resource.accounts_count
-    @domain_block   = resource.is_a?(DomainBlock) ? resource : DomainBlock.find_by(domain: domain)
+    @domain_block   = resource.is_a?(DomainBlock) ? resource : DomainBlock.rule_for(domain)
   end
 
   def cached_sample_accounts
diff --git a/app/services/activitypub/process_account_service.rb b/app/services/activitypub/process_account_service.rb
index ad22d37fe..05c017bdf 100644
--- a/app/services/activitypub/process_account_service.rb
+++ b/app/services/activitypub/process_account_service.rb
@@ -205,7 +205,7 @@ class ActivityPub::ProcessAccountService < BaseService
 
   def domain_block
     return @domain_block if defined?(@domain_block)
-    @domain_block = DomainBlock.find_by(domain: @domain)
+    @domain_block = DomainBlock.rule_for(@domain)
   end
 
   def key_changed?
diff --git a/app/services/block_domain_service.rb b/app/services/block_domain_service.rb
index 497f0394b..c6eef04d4 100644
--- a/app/services/block_domain_service.rb
+++ b/app/services/block_domain_service.rb
@@ -76,7 +76,7 @@ class BlockDomainService < BaseService
   end
 
   def blocked_domain_accounts
-    Account.where(domain: blocked_domain)
+    Account.by_domain_and_subdomains(blocked_domain)
   end
 
   def media_from_blocked_domain
@@ -84,6 +84,6 @@ class BlockDomainService < BaseService
   end
 
   def emojis_from_blocked_domains
-    CustomEmoji.where(domain: blocked_domain)
+    CustomEmoji.by_domain_and_subdomains(blocked_domain)
   end
 end
diff --git a/app/services/resolve_account_service.rb b/app/services/resolve_account_service.rb
index 11e33a83a..57c9ccfe1 100644
--- a/app/services/resolve_account_service.rb
+++ b/app/services/resolve_account_service.rb
@@ -146,7 +146,7 @@ class ResolveAccountService < BaseService
 
   def domain_block
     return @domain_block if defined?(@domain_block)
-    @domain_block = DomainBlock.find_by(domain: @domain)
+    @domain_block = DomainBlock.rule_for(@domain)
   end
 
   def atom_url
diff --git a/app/services/unblock_domain_service.rb b/app/services/unblock_domain_service.rb
index 9b8526fbe..fc262a50a 100644
--- a/app/services/unblock_domain_service.rb
+++ b/app/services/unblock_domain_service.rb
@@ -14,7 +14,8 @@ class UnblockDomainService < BaseService
   end
 
   def blocked_accounts
-    scope = Account.where(domain: domain_block.domain)
+    scope = Account.by_domain_and_subdomains(domain_block.domain)
+
     if domain_block.silence?
       scope.where(silenced_at: @domain_block.created_at)
     else
diff --git a/app/services/update_remote_profile_service.rb b/app/services/update_remote_profile_service.rb
index 68d36addf..403395a0d 100644
--- a/app/services/update_remote_profile_service.rb
+++ b/app/services/update_remote_profile_service.rb
@@ -26,7 +26,7 @@ class UpdateRemoteProfileService < BaseService
     account.note         = remote_profile.note         || ''
     account.locked       = remote_profile.locked?
 
-    if !account.suspended? && !DomainBlock.find_by(domain: account.domain)&.reject_media?
+    if !account.suspended? && !DomainBlock.reject_media?(account.domain)
       if remote_profile.avatar.present?
         account.avatar_remote_url = remote_profile.avatar
       else
@@ -46,7 +46,7 @@ class UpdateRemoteProfileService < BaseService
   end
 
   def save_emojis
-    do_not_download = DomainBlock.find_by(domain: account.domain)&.reject_media?
+    do_not_download = DomainBlock.reject_media?(account.domain)
 
     return if do_not_download
 
diff --git a/spec/models/account_spec.rb b/spec/models/account_spec.rb
index 379872316..ce9ea250d 100644
--- a/spec/models/account_spec.rb
+++ b/spec/models/account_spec.rb
@@ -687,6 +687,23 @@ RSpec.describe Account, type: :model do
       end
     end
 
+    describe 'by_domain_and_subdomains' do
+      it 'returns exact domain matches' do
+        account = Fabricate(:account, domain: 'example.com')
+        expect(Account.by_domain_and_subdomains('example.com')).to eq [account]
+      end
+
+      it 'returns subdomains' do
+        account = Fabricate(:account, domain: 'foo.example.com')
+        expect(Account.by_domain_and_subdomains('example.com')).to eq [account]
+      end
+
+      it 'does not return partially matching domains' do
+        account = Fabricate(:account, domain: 'grexample.com')
+        expect(Account.by_domain_and_subdomains('example.com')).to_not eq [account]
+      end
+    end
+
     describe 'expiring' do
       it 'returns remote accounts with followers whose subscription expiration date is past or not given' do
         local = Fabricate(:account, domain: nil)
diff --git a/spec/models/domain_block_spec.rb b/spec/models/domain_block_spec.rb
index 0035fd0ff..d98c5e118 100644
--- a/spec/models/domain_block_spec.rb
+++ b/spec/models/domain_block_spec.rb
@@ -21,23 +21,40 @@ RSpec.describe DomainBlock, type: :model do
     end
   end
 
-  describe 'blocked?' do
+  describe '.blocked?' do
     it 'returns true if the domain is suspended' do
-      Fabricate(:domain_block, domain: 'domain', severity: :suspend)
-      expect(DomainBlock.blocked?('domain')).to eq true
+      Fabricate(:domain_block, domain: 'example.com', severity: :suspend)
+      expect(DomainBlock.blocked?('example.com')).to eq true
     end
 
     it 'returns false even if the domain is silenced' do
-      Fabricate(:domain_block, domain: 'domain', severity: :silence)
-      expect(DomainBlock.blocked?('domain')).to eq false
+      Fabricate(:domain_block, domain: 'example.com', severity: :silence)
+      expect(DomainBlock.blocked?('example.com')).to eq false
     end
 
     it 'returns false if the domain is not suspended nor silenced' do
-      expect(DomainBlock.blocked?('domain')).to eq false
+      expect(DomainBlock.blocked?('example.com')).to eq false
     end
   end
 
-  describe 'stricter_than?' do
+  describe '.rule_for' do
+    it 'returns rule matching a blocked domain' do
+      block = Fabricate(:domain_block, domain: 'example.com')
+      expect(DomainBlock.rule_for('example.com')).to eq block
+    end
+
+    it 'returns a rule matching a subdomain of a blocked domain' do
+      block = Fabricate(:domain_block, domain: 'example.com')
+      expect(DomainBlock.rule_for('sub.example.com')).to eq block
+    end
+
+    it 'returns a rule matching a blocked subdomain' do
+      block = Fabricate(:domain_block, domain: 'sub.example.com')
+      expect(DomainBlock.rule_for('sub.example.com')).to eq block
+    end
+  end
+
+  describe '#stricter_than?' do
     it 'returns true if the new block has suspend severity while the old has lower severity' do
       suspend = DomainBlock.new(domain: 'domain', severity: :suspend)
       silence = DomainBlock.new(domain: 'domain', severity: :silence)
-- 
cgit 


From 1b052c7b2d70b371648732d4d2a1d0ca3a52cab0 Mon Sep 17 00:00:00 2001
From: Eugen Rochko <eugen@zeonfederated.com>
Date: Sat, 22 Jun 2019 00:39:09 +0200
Subject: Remove expensive counters from federation page in admin UI (#11139)

---
 app/models/instance.rb                    |  8 ++------
 app/views/admin/instances/index.html.haml | 21 +++++++++++----------
 2 files changed, 13 insertions(+), 16 deletions(-)

(limited to 'app/models')

diff --git a/app/models/instance.rb b/app/models/instance.rb
index a01db1212..797a191e0 100644
--- a/app/models/instance.rb
+++ b/app/models/instance.rb
@@ -11,12 +11,8 @@ class Instance
     @domain_block   = resource.is_a?(DomainBlock) ? resource : DomainBlock.rule_for(domain)
   end
 
-  def cached_sample_accounts
-    Rails.cache.fetch("#{cache_key}/sample_accounts", expires_in: 12.hours) { Account.where(domain: domain).searchable.joins(:account_stat).popular.limit(3) }
-  end
-
-  def cached_accounts_count
-    @accounts_count || Rails.cache.fetch("#{cache_key}/count", expires_in: 12.hours) { Account.where(domain: domain).count }
+  def countable?
+    @accounts_count.present?
   end
 
   def to_param
diff --git a/app/views/admin/instances/index.html.haml b/app/views/admin/instances/index.html.haml
index 9574c3147..61e578409 100644
--- a/app/views/admin/instances/index.html.haml
+++ b/app/views/admin/instances/index.html.haml
@@ -33,21 +33,22 @@
       %h4
         = instance.domain
         %small
-          = t('admin.instances.known_accounts', count: instance.cached_accounts_count)
-
           - if instance.domain_block
+            - first_item = true
             - if !instance.domain_block.noop?
-              &bull;
               = t("admin.domain_blocks.severity.#{instance.domain_block.severity}")
+              - first_item = false
             - if instance.domain_block.reject_media?
-              &bull;
+              - unless first_item
+                &bull;
               = t('admin.domain_blocks.rejecting_media')
+              - first_item = false
             - if instance.domain_block.reject_reports?
-              &bull;
+              - unless first_item
+                &bull;
               = t('admin.domain_blocks.rejecting_reports')
-
-      .avatar-stack
-        - instance.cached_sample_accounts.each do |account|
-          = image_tag current_account&.user&.setting_auto_play_gif ? account.avatar_original_url : account.avatar_static_url, width: 48, height: 48, alt: '', class: 'account__avatar'
-
+          - else
+            = t('admin.accounts.no_limits_imposed')
+      - if instance.countable?
+        .trends__item__current{ title: t('admin.instances.known_accounts', count: instance.accounts_count) }= number_to_human instance.accounts_count, strip_insignificant_zeros: true
 = paginate paginated_instances
-- 
cgit 


From aa9b37822bd39dff771844b7b1ac149e37788d34 Mon Sep 17 00:00:00 2001
From: Eugen Rochko <eugen@zeonfederated.com>
Date: Sat, 22 Jun 2019 02:50:36 +0200
Subject: Fix audio not being downloaded from remote servers (#11145)

---
 app/lib/activitypub/activity/create.rb      | 2 +-
 app/models/media_attachment.rb              | 8 ++++++++
 app/serializers/initial_state_serializer.rb | 2 +-
 3 files changed, 10 insertions(+), 2 deletions(-)

(limited to 'app/models')

diff --git a/app/lib/activitypub/activity/create.rb b/app/lib/activitypub/activity/create.rb
index 487e8e91e..00f0dd42d 100644
--- a/app/lib/activitypub/activity/create.rb
+++ b/app/lib/activitypub/activity/create.rb
@@ -370,7 +370,7 @@ class ActivityPub::Activity::Create < ActivityPub::Activity
   end
 
   def unsupported_media_type?(mime_type)
-    mime_type.present? && !(MediaAttachment::IMAGE_MIME_TYPES + MediaAttachment::VIDEO_MIME_TYPES).include?(mime_type)
+    mime_type.present? && !MediaAttachment.supported_mime_types.include?(mime_type)
   end
 
   def supported_blurhash?(blurhash)
diff --git a/app/models/media_attachment.rb b/app/models/media_attachment.rb
index 4c55d4b98..a9b6d96c6 100644
--- a/app/models/media_attachment.rb
+++ b/app/models/media_attachment.rb
@@ -177,6 +177,14 @@ class MediaAttachment < ApplicationRecord
   before_save :set_meta
 
   class << self
+    def supported_mime_types
+      IMAGE_MIME_TYPES + VIDEO_MIME_TYPES + AUDIO_MIME_TYPES
+    end
+
+    def supported_file_extensions
+      IMAGE_FILE_EXTENSIONS + VIDEO_FILE_EXTENSIONS + AUDIO_FILE_EXTENSIONS
+    end
+
     private
 
     def file_styles(f)
diff --git a/app/serializers/initial_state_serializer.rb b/app/serializers/initial_state_serializer.rb
index 3d825410e..8daf36e0a 100644
--- a/app/serializers/initial_state_serializer.rb
+++ b/app/serializers/initial_state_serializer.rb
@@ -60,7 +60,7 @@ class InitialStateSerializer < ActiveModel::Serializer
   end
 
   def media_attachments
-    { accept_content_types: MediaAttachment::IMAGE_FILE_EXTENSIONS + MediaAttachment::VIDEO_FILE_EXTENSIONS + MediaAttachment::AUDIO_FILE_EXTENSIONS + MediaAttachment::IMAGE_MIME_TYPES + MediaAttachment::VIDEO_MIME_TYPES + MediaAttachment::AUDIO_MIME_TYPES }
+    { accept_content_types: MediaAttachment.supported_file_extensions + MediaAttachment.supported_mime_types }
   end
 
   private
-- 
cgit 


From b927bb3f07b88d0a7bc39a7661d26dd1c9fc05d4 Mon Sep 17 00:00:00 2001
From: Eugen Rochko <eugen@zeonfederated.com>
Date: Sat, 22 Jun 2019 16:54:06 +0200
Subject: Fix audio-only OGG and WebM files not being processed as such
 (#11151)

Also, because Chrome sends audio/mp3 instead of audio/mpeg as it's
supposed to, we need to whitelist that mime type as well
---
 app/models/concerns/attachmentable.rb | 17 +++++++++++++++++
 app/models/media_attachment.rb        |  4 ++--
 2 files changed, 19 insertions(+), 2 deletions(-)

(limited to 'app/models')

diff --git a/app/models/concerns/attachmentable.rb b/app/models/concerns/attachmentable.rb
index f4e37f1e6..24f5968de 100644
--- a/app/models/concerns/attachmentable.rb
+++ b/app/models/concerns/attachmentable.rb
@@ -10,10 +10,21 @@ module Attachmentable
   included do
     before_post_process :set_file_extensions
     before_post_process :check_image_dimensions
+    before_post_process :set_file_content_type
   end
 
   private
 
+  def set_file_content_type
+    self.class.attachment_definitions.each_key do |attachment_name|
+      attachment = send(attachment_name)
+
+      next if attachment.blank? || attachment.queued_for_write[:original].blank?
+
+      attachment.instance_write :content_type, calculated_content_type(attachment)
+    end
+  end
+
   def set_file_extensions
     self.class.attachment_definitions.each_key do |attachment_name|
       attachment = send(attachment_name)
@@ -47,4 +58,10 @@ module Attachmentable
 
     extension
   end
+
+  def calculated_content_type(attachment)
+    Paperclip.run('file', '-b --mime :file', file: attachment.queued_for_write[:original].path).split(/[:;\s]+/).first.chomp
+  rescue Terrapin::CommandLineError
+    ''
+  end
 end
diff --git a/app/models/media_attachment.rb b/app/models/media_attachment.rb
index a9b6d96c6..30d9a9851 100644
--- a/app/models/media_attachment.rb
+++ b/app/models/media_attachment.rb
@@ -31,9 +31,9 @@ class MediaAttachment < ApplicationRecord
   AUDIO_FILE_EXTENSIONS = ['.ogg', '.oga', '.mp3', '.wav', '.flac', '.opus'].freeze
 
   IMAGE_MIME_TYPES             = ['image/jpeg', 'image/png', 'image/gif', 'image/webp'].freeze
-  VIDEO_MIME_TYPES             = ['video/webm', 'video/mp4', 'video/quicktime'].freeze
+  VIDEO_MIME_TYPES             = ['video/webm', 'video/mp4', 'video/quicktime', 'video/ogg'].freeze
   VIDEO_CONVERTIBLE_MIME_TYPES = ['video/webm', 'video/quicktime'].freeze
-  AUDIO_MIME_TYPES             = ['audio/wave', 'audio/wav', 'audio/x-wav', 'audio/x-pn-wave', 'audio/ogg', 'audio/mpeg', 'audio/webm', 'audio/flac'].freeze
+  AUDIO_MIME_TYPES             = ['audio/wave', 'audio/wav', 'audio/x-wav', 'audio/x-pn-wave', 'audio/ogg', 'audio/mpeg', 'audio/mp3', 'audio/webm', 'audio/flac'].freeze
 
   BLURHASH_OPTIONS = {
     x_comp: 4,
-- 
cgit 


From 3086c645fde2345d34e401bdf3e2f19f19da3294 Mon Sep 17 00:00:00 2001
From: ThibG <thib@sitedethib.com>
Date: Wed, 26 Jun 2019 19:33:04 +0200
Subject: Add option to disable blurhash previews (#11188)

* Add option to disable blurhash previews

* Update option text

* Change options order
---
 app/controllers/settings/preferences_controller.rb       | 1 +
 app/javascript/mastodon/components/media_gallery.js      | 4 +++-
 app/javascript/mastodon/features/video/index.js          | 4 +++-
 app/javascript/mastodon/initial_state.js                 | 1 +
 app/lib/user_settings_decorator.rb                       | 5 +++++
 app/models/user.rb                                       | 2 +-
 app/serializers/initial_state_serializer.rb              | 1 +
 app/views/settings/preferences/appearance/show.html.haml | 3 +++
 config/locales/simple_form.en.yml                        | 2 ++
 config/settings.yml                                      | 1 +
 10 files changed, 21 insertions(+), 3 deletions(-)

(limited to 'app/models')

diff --git a/app/controllers/settings/preferences_controller.rb b/app/controllers/settings/preferences_controller.rb
index 110debd6e..0a5c14cca 100644
--- a/app/controllers/settings/preferences_controller.rb
+++ b/app/controllers/settings/preferences_controller.rb
@@ -54,6 +54,7 @@ class Settings::PreferencesController < Settings::BaseController
       :setting_aggregate_reblogs,
       :setting_show_application,
       :setting_advanced_layout,
+      :setting_use_blurhash,
       notification_emails: %i(follow follow_request reblog favourite mention digest report pending_account),
       interactions: %i(must_be_follower must_be_following must_be_following_dm)
     )
diff --git a/app/javascript/mastodon/components/media_gallery.js b/app/javascript/mastodon/components/media_gallery.js
index 77bac61ee..9cd71b7c9 100644
--- a/app/javascript/mastodon/components/media_gallery.js
+++ b/app/javascript/mastodon/components/media_gallery.js
@@ -6,7 +6,7 @@ import IconButton from './icon_button';
 import { defineMessages, injectIntl, FormattedMessage } from 'react-intl';
 import { isIOS } from '../is_mobile';
 import classNames from 'classnames';
-import { autoPlayGif, displayMedia } from '../initial_state';
+import { autoPlayGif, displayMedia, useBlurhash } from '../initial_state';
 import { decode } from 'blurhash';
 
 const messages = defineMessages({
@@ -81,6 +81,8 @@ class Item extends React.PureComponent {
   }
 
   _decode () {
+    if (!useBlurhash) return;
+
     const hash   = this.props.attachment.get('blurhash');
     const pixels = decode(hash, 32, 32);
 
diff --git a/app/javascript/mastodon/features/video/index.js b/app/javascript/mastodon/features/video/index.js
index b0c408527..0acdd198d 100644
--- a/app/javascript/mastodon/features/video/index.js
+++ b/app/javascript/mastodon/features/video/index.js
@@ -5,7 +5,7 @@ import { fromJS, is } from 'immutable';
 import { throttle } from 'lodash';
 import classNames from 'classnames';
 import { isFullscreen, requestFullscreen, exitFullscreen } from '../ui/util/fullscreen';
-import { displayMedia } from '../../initial_state';
+import { displayMedia, useBlurhash } from '../../initial_state';
 import Icon from 'mastodon/components/icon';
 import { decode } from 'blurhash';
 
@@ -298,6 +298,8 @@ class Video extends React.PureComponent {
   }
 
   _decode () {
+    if (!useBlurhash) return;
+
     const hash   = this.props.blurhash;
     const pixels = decode(hash, 32, 32);
 
diff --git a/app/javascript/mastodon/initial_state.js b/app/javascript/mastodon/initial_state.js
index 125508c23..f0f131ff5 100644
--- a/app/javascript/mastodon/initial_state.js
+++ b/app/javascript/mastodon/initial_state.js
@@ -20,5 +20,6 @@ export const mascot = getMeta('mascot');
 export const profile_directory = getMeta('profile_directory');
 export const isStaff = getMeta('is_staff');
 export const forceSingleColumn = !getMeta('advanced_layout');
+export const useBlurhash = getMeta('use_blurhash');
 
 export default initialState;
diff --git a/app/lib/user_settings_decorator.rb b/app/lib/user_settings_decorator.rb
index bf2e5a962..aaf95cc19 100644
--- a/app/lib/user_settings_decorator.rb
+++ b/app/lib/user_settings_decorator.rb
@@ -34,6 +34,7 @@ class UserSettingsDecorator
     user.settings['aggregate_reblogs']   = aggregate_reblogs_preference if change?('setting_aggregate_reblogs')
     user.settings['show_application']    = show_application_preference if change?('setting_show_application')
     user.settings['advanced_layout']     = advanced_layout_preference if change?('setting_advanced_layout')
+    user.settings['use_blurhash']        = use_blurhash_preference if change?('setting_use_blurhash')
   end
 
   def merged_notification_emails
@@ -112,6 +113,10 @@ class UserSettingsDecorator
     boolean_cast_setting 'setting_advanced_layout'
   end
 
+  def use_blurhash_preference
+    boolean_cast_setting 'setting_use_blurhash'
+  end
+
   def boolean_cast_setting(key)
     ActiveModel::Type::Boolean.new.cast(settings[key])
   end
diff --git a/app/models/user.rb b/app/models/user.rb
index 50873dd01..3a4b415dd 100644
--- a/app/models/user.rb
+++ b/app/models/user.rb
@@ -106,7 +106,7 @@ class User < ApplicationRecord
   delegate :auto_play_gif, :default_sensitive, :unfollow_modal, :boost_modal, :delete_modal,
            :reduce_motion, :system_font_ui, :noindex, :theme, :display_media, :hide_network,
            :expand_spoilers, :default_language, :aggregate_reblogs, :show_application,
-           :advanced_layout, to: :settings, prefix: :setting, allow_nil: false
+           :advanced_layout, :use_blurhash, to: :settings, prefix: :setting, allow_nil: false
 
   attr_reader :invite_code
   attr_writer :external
diff --git a/app/serializers/initial_state_serializer.rb b/app/serializers/initial_state_serializer.rb
index 8daf36e0a..54f99d570 100644
--- a/app/serializers/initial_state_serializer.rb
+++ b/app/serializers/initial_state_serializer.rb
@@ -32,6 +32,7 @@ class InitialStateSerializer < ActiveModel::Serializer
       store[:expand_spoilers] = object.current_account.user.setting_expand_spoilers
       store[:reduce_motion]   = object.current_account.user.setting_reduce_motion
       store[:advanced_layout] = object.current_account.user.setting_advanced_layout
+      store[:use_blurhash]    = object.current_account.user.setting_use_blurhash
       store[:is_staff]        = object.current_account.user.staff?
     end
 
diff --git a/app/views/settings/preferences/appearance/show.html.haml b/app/views/settings/preferences/appearance/show.html.haml
index 10f009264..c13882801 100644
--- a/app/views/settings/preferences/appearance/show.html.haml
+++ b/app/views/settings/preferences/appearance/show.html.haml
@@ -34,6 +34,9 @@
   .fields-group
     = f.input :setting_display_media, collection: ['default', 'show_all', 'hide_all'],label_method: lambda { |item| t("simple_form.hints.defaults.setting_display_media_#{item}") }, hint: false, as: :radio_buttons, collection_wrapper_tag: 'ul', item_wrapper_tag: 'li', wrapper: :with_floating_label
 
+  .fields-group
+    = f.input :setting_use_blurhash, as: :boolean, wrapper: :with_label
+
   .fields-group
     = f.input :setting_expand_spoilers, as: :boolean, wrapper: :with_label
 
diff --git a/config/locales/simple_form.en.yml b/config/locales/simple_form.en.yml
index 4602f9cd9..75cbec9de 100644
--- a/config/locales/simple_form.en.yml
+++ b/config/locales/simple_form.en.yml
@@ -34,6 +34,7 @@ en:
         setting_hide_network: Who you follow and who follows you will not be shown on your profile
         setting_noindex: Affects your public profile and status pages
         setting_show_application: The application you use to toot will be displayed in the detailed view of your toots
+        setting_use_blurhash: Gradients are based on the colors of the hidden visuals but obfuscate any details
         username: Your username will be unique on %{domain}
         whole_word: When the keyword or phrase is alphanumeric only, it will only be applied if it matches the whole word
       featured_tag:
@@ -109,6 +110,7 @@ en:
         setting_system_font_ui: Use system's default font
         setting_theme: Site theme
         setting_unfollow_modal: Show confirmation dialog before unfollowing someone
+        setting_use_blurhash: Show colorful gradients for hidden media
         severity: Severity
         type: Import type
         username: Username
diff --git a/config/settings.yml b/config/settings.yml
index 805624d3e..ad71b6008 100644
--- a/config/settings.yml
+++ b/config/settings.yml
@@ -32,6 +32,7 @@ defaults: &defaults
   theme: 'default'
   aggregate_reblogs: true
   advanced_layout: false
+  use_blurhash: true
   notification_emails:
     follow: false
     reblog: false
-- 
cgit 


From ca17bae904783dfb1f4899a533d28a79da0c6fe9 Mon Sep 17 00:00:00 2001
From: Thibaut Girka <thib@sitedethib.com>
Date: Tue, 25 Jun 2019 22:56:32 +0200
Subject: Use a redis-cached feed for the DM timeline

---
 .../api/v1/timelines/direct_controller.rb          | 10 +++---
 app/lib/feed_manager.rb                            | 40 ++++++++++++++++++++++
 app/models/direct_feed.rb                          | 31 +++++++++++++++++
 app/services/batched_remove_status_service.rb      |  4 +--
 app/services/fan_out_on_write_service.rb           |  7 ++--
 app/services/precompute_feed_service.rb            |  1 +
 app/services/remove_status_service.rb              |  4 +--
 app/workers/feed_insert_worker.rb                  | 11 +++++-
 app/workers/scheduler/feed_cleanup_scheduler.rb    |  5 +++
 9 files changed, 100 insertions(+), 13 deletions(-)
 create mode 100644 app/models/direct_feed.rb

(limited to 'app/models')

diff --git a/app/controllers/api/v1/timelines/direct_controller.rb b/app/controllers/api/v1/timelines/direct_controller.rb
index d8a76d153..6e98e9cac 100644
--- a/app/controllers/api/v1/timelines/direct_controller.rb
+++ b/app/controllers/api/v1/timelines/direct_controller.rb
@@ -27,16 +27,18 @@ class Api::V1::Timelines::DirectController < Api::BaseController
   end
 
   def direct_timeline_statuses
-    # this query requires built in pagination.
-    Status.as_direct_timeline(
-      current_account,
+    account_direct_feed.get(
       limit_param(DEFAULT_STATUSES_LIMIT),
       params[:max_id],
       params[:since_id],
-      true # returns array of cache_ids object
+      params[:min_id]
     )
   end
 
+  def account_direct_feed
+    DirectFeed.new(current_account)
+  end
+
   def insert_pagination_headers
     set_pagination_headers(next_path, prev_path)
   end
diff --git a/app/lib/feed_manager.rb b/app/lib/feed_manager.rb
index ddcbdf6da..59767cdfe 100644
--- a/app/lib/feed_manager.rb
+++ b/app/lib/feed_manager.rb
@@ -22,6 +22,8 @@ class FeedManager
       filter_from_home?(status, receiver_id)
     elsif timeline_type == :mentions
       filter_from_mentions?(status, receiver_id)
+    elsif timeline_type == :direct
+      filter_from_direct?(status, receiver_id)
     else
       false
     end
@@ -59,6 +61,18 @@ class FeedManager
     true
   end
 
+  def push_to_direct(account, status)
+    return false unless add_to_feed(:direct, account.id, status)
+    trim(:direct, account.id)
+    PushUpdateWorker.perform_async(account.id, status.id, "timeline:direct:#{account.id}")
+    true
+  end
+
+  def unpush_from_direct(account, status)
+    return false unless remove_from_feed(:direct, account.id, status)
+    redis.publish("timeline:direct:#{account.id}", Oj.dump(event: :delete, payload: status.id.to_s))
+  end
+
   def trim(type, account_id)
     timeline_key = key(type, account_id)
     reblog_key   = key(type, account_id, 'reblogs')
@@ -142,6 +156,27 @@ class FeedManager
     end
   end
 
+  def populate_direct_feed(account)
+    added  = 0
+    limit  = FeedManager::MAX_ITEMS / 2
+    max_id = nil
+
+    loop do
+      statuses = Status.as_direct_timeline(account, limit, max_id)
+
+      break if statuses.empty?
+
+      statuses.each do |status|
+        next if filter_from_direct?(status, account)
+        added += 1 if add_to_feed(:direct, account.id, status)
+      end
+
+      break unless added.zero?
+
+      max_id = statuses.last.id
+    end
+  end
+
   private
 
   def push_update_required?(timeline_id)
@@ -199,6 +234,11 @@ class FeedManager
     should_filter
   end
 
+  def filter_from_direct?(status, receiver_id)
+    return false if receiver_id == status.account_id
+    filter_from_mentions?(status, receiver_id)
+  end
+
   def phrase_filtered?(status, receiver_id, context)
     active_filters = Rails.cache.fetch("filters:#{receiver_id}") { CustomFilter.where(account_id: receiver_id).active_irreversible.to_a }.to_a
 
diff --git a/app/models/direct_feed.rb b/app/models/direct_feed.rb
new file mode 100644
index 000000000..c0b8a0a35
--- /dev/null
+++ b/app/models/direct_feed.rb
@@ -0,0 +1,31 @@
+# frozen_string_literal: true
+
+class DirectFeed < Feed
+  include Redisable
+
+  def initialize(account)
+    @type    = :direct
+    @id      = account.id
+    @account = account
+  end
+
+  def get(limit, max_id = nil, since_id = nil, min_id = nil)
+    unless redis.exists("account:#{@account.id}:regeneration")
+      statuses = super
+      return statuses unless statuses.empty?
+    end
+    from_database(limit, max_id, since_id, min_id)
+  end
+
+  private
+
+  def from_database(limit, max_id, since_id, min_id)
+    loop do
+      statuses = Status.as_direct_timeline(@account, limit, max_id, since_id, min_id)
+      return statuses if statuses.empty?
+      max_id = statuses.last.id
+      statuses = statuses.reject { |status| FeedManager.instance.filter?(:direct, status, @account.id) }
+      return statuses unless statuses.empty?
+    end
+  end
+end
diff --git a/app/services/batched_remove_status_service.rb b/app/services/batched_remove_status_service.rb
index 02f7076f7..2fe009c91 100644
--- a/app/services/batched_remove_status_service.rb
+++ b/app/services/batched_remove_status_service.rb
@@ -107,9 +107,9 @@ class BatchedRemoveStatusService < BaseService
     payload = @json_payloads[status.id]
     redis.pipelined do
       @mentions[status.id].each do |mention|
-        redis.publish("timeline:direct:#{mention.account.id}", payload) if mention.account.local?
+        FeedManager.instance.unpush_from_direct(mention.account, status) if mention.account.local?
       end
-      redis.publish("timeline:direct:#{status.account.id}", payload) if status.account.local?
+      FeedManager.instance.unpush_from_direct(status.account, status) if status.account.local?
     end
   end
 
diff --git a/app/services/fan_out_on_write_service.rb b/app/services/fan_out_on_write_service.rb
index b66dc342e..cf433d8a6 100644
--- a/app/services/fan_out_on_write_service.rb
+++ b/app/services/fan_out_on_write_service.rb
@@ -37,6 +37,7 @@ class FanOutOnWriteService < BaseService
   def deliver_to_self(status)
     Rails.logger.debug "Delivering status #{status.id} to author"
     FeedManager.instance.push_to_home(status.account, status)
+    FeedManager.instance.push_to_direct(status.account, status) if status.direct_visibility?
   end
 
   def deliver_to_followers(status)
@@ -98,11 +99,9 @@ class FanOutOnWriteService < BaseService
   def deliver_to_direct_timelines(status)
     Rails.logger.debug "Delivering status #{status.id} to direct timelines"
 
-    status.mentions.includes(:account).each do |mention|
-      Redis.current.publish("timeline:direct:#{mention.account.id}", @payload) if mention.account.local?
+    FeedInsertWorker.push_bulk(status.mentions.includes(:account).map(&:account).select { |mentioned_account| mentioned_account.local? }) do |account|
+      [status.id, account.id, :direct]
     end
-
-    Redis.current.publish("timeline:direct:#{status.account.id}", @payload) if status.account.local?
   end
 
   def deliver_to_own_conversation(status)
diff --git a/app/services/precompute_feed_service.rb b/app/services/precompute_feed_service.rb
index 076dedaca..029c2f6e5 100644
--- a/app/services/precompute_feed_service.rb
+++ b/app/services/precompute_feed_service.rb
@@ -3,6 +3,7 @@
 class PrecomputeFeedService < BaseService
   def call(account)
     FeedManager.instance.populate_feed(account)
+    FeedManager.instance.populate_direct_feed(account)
   ensure
     Redis.current.del("account:#{account.id}:regeneration")
   end
diff --git a/app/services/remove_status_service.rb b/app/services/remove_status_service.rb
index 98972fc70..9d5d0fc14 100644
--- a/app/services/remove_status_service.rb
+++ b/app/services/remove_status_service.rb
@@ -48,6 +48,7 @@ class RemoveStatusService < BaseService
 
   def remove_from_self
     FeedManager.instance.unpush_from_home(@account, @status)
+    FeedManager.instance.unpush_from_direct(@account, @status) if @status.direct_visibility?
   end
 
   def remove_from_followers
@@ -159,9 +160,8 @@ class RemoveStatusService < BaseService
 
   def remove_from_direct
     @mentions.each do |mention|
-      Redis.current.publish("timeline:direct:#{mention.account.id}", @payload) if mention.account.local?
+      FeedManager.instance.unpush_from_direct(mention.account, @status) if mention.account.local?
     end
-    Redis.current.publish("timeline:direct:#{@account.id}", @payload) if @account.local?
   end
 
   def lock_options
diff --git a/app/workers/feed_insert_worker.rb b/app/workers/feed_insert_worker.rb
index 1ae3c877b..546f5c0c2 100644
--- a/app/workers/feed_insert_worker.rb
+++ b/app/workers/feed_insert_worker.rb
@@ -13,6 +13,8 @@ class FeedInsertWorker
     when :list
       @list     = List.find(id)
       @follower = @list.account
+    when :direct
+      @account  = Account.find(id)
     end
 
     check_and_insert
@@ -29,7 +31,12 @@ class FeedInsertWorker
   def feed_filtered?
     # Note: Lists are a variation of home, so the filtering rules
     # of home apply to both
-    FeedManager.instance.filter?(:home, @status, @follower.id)
+    case @type
+    when :home, :list
+      FeedManager.instance.filter?(:home, @status, @follower.id)
+    when :direct
+      FeedManager.instance.filter?(:direct, @status, @account.id)
+    end
   end
 
   def perform_push
@@ -38,6 +45,8 @@ class FeedInsertWorker
       FeedManager.instance.push_to_home(@follower, @status)
     when :list
       FeedManager.instance.push_to_list(@list, @status)
+    when :direct
+      FeedManager.instance.push_to_direct(@account, @status)
     end
   end
 end
diff --git a/app/workers/scheduler/feed_cleanup_scheduler.rb b/app/workers/scheduler/feed_cleanup_scheduler.rb
index bf5e20757..4933f1753 100644
--- a/app/workers/scheduler/feed_cleanup_scheduler.rb
+++ b/app/workers/scheduler/feed_cleanup_scheduler.rb
@@ -9,6 +9,7 @@ class Scheduler::FeedCleanupScheduler
   def perform
     clean_home_feeds!
     clean_list_feeds!
+    clean_direct_feeds!
   end
 
   private
@@ -21,6 +22,10 @@ class Scheduler::FeedCleanupScheduler
     clean_feeds!(inactive_list_ids, :list)
   end
 
+  def clean_direct_feeds!
+    clean_feeds!(inactive_account_ids, :direct)
+  end
+
   def clean_feeds!(ids, type)
     reblogged_id_sets = {}
 
-- 
cgit 


From e64e6a03dd1e0978fee48f0596dcfbc7fd29958f Mon Sep 17 00:00:00 2001
From: Eugen Rochko <eugen@zeonfederated.com>
Date: Fri, 28 Jun 2019 15:54:10 +0200
Subject: Add categories for custom emojis (#11196)

Fix #7940
---
 app/controllers/api/v1/custom_emojis_controller.rb |  2 +-
 .../compose/components/emoji_picker_dropdown.js    | 30 ++++++++++++----------
 app/javascript/mastodon/features/emoji/emoji.js    |  3 +++
 app/models/custom_emoji.rb                         |  2 ++
 app/models/custom_emoji_category.rb                | 15 +++++++++++
 app/serializers/rest/custom_emoji_serializer.rb    | 10 ++++++++
 ...0190627222225_create_custom_emoji_categories.rb |  9 +++++++
 ...90627222826_add_category_id_to_custom_emojis.rb |  5 ++++
 db/schema.rb                                       | 10 +++++++-
 lib/mastodon/emoji_cli.rb                          |  6 +++++
 .../custom_emoji_category_fabricator.rb            |  3 +++
 spec/models/custom_emoji_category_spec.rb          |  5 ++++
 yarn.lock                                          |  4 +--
 13 files changed, 86 insertions(+), 18 deletions(-)
 create mode 100644 app/models/custom_emoji_category.rb
 create mode 100644 db/migrate/20190627222225_create_custom_emoji_categories.rb
 create mode 100644 db/migrate/20190627222826_add_category_id_to_custom_emojis.rb
 create mode 100644 spec/fabricators/custom_emoji_category_fabricator.rb
 create mode 100644 spec/models/custom_emoji_category_spec.rb

(limited to 'app/models')

diff --git a/app/controllers/api/v1/custom_emojis_controller.rb b/app/controllers/api/v1/custom_emojis_controller.rb
index 1bb19a09d..b6877fb3c 100644
--- a/app/controllers/api/v1/custom_emojis_controller.rb
+++ b/app/controllers/api/v1/custom_emojis_controller.rb
@@ -7,7 +7,7 @@ class Api::V1::CustomEmojisController < Api::BaseController
 
   def index
     render_cached_json('api:v1:custom_emojis', expires_in: 1.minute) do
-      ActiveModelSerializers::SerializableResource.new(CustomEmoji.local.where(disabled: false), each_serializer: REST::CustomEmojiSerializer)
+      ActiveModelSerializers::SerializableResource.new(CustomEmoji.local.where(disabled: false).includes(:category), each_serializer: REST::CustomEmojiSerializer)
     end
   end
 end
diff --git a/app/javascript/mastodon/features/compose/components/emoji_picker_dropdown.js b/app/javascript/mastodon/features/compose/components/emoji_picker_dropdown.js
index c1429c756..e57c3c20c 100644
--- a/app/javascript/mastodon/features/compose/components/emoji_picker_dropdown.js
+++ b/app/javascript/mastodon/features/compose/components/emoji_picker_dropdown.js
@@ -6,7 +6,7 @@ import Overlay from 'react-overlays/lib/Overlay';
 import classNames from 'classnames';
 import ImmutablePropTypes from 'react-immutable-proptypes';
 import detectPassiveEvents from 'detect-passive-events';
-import { buildCustomEmojis } from '../../emoji/emoji';
+import { buildCustomEmojis, categoriesFromEmojis } from '../../emoji/emoji';
 
 const messages = defineMessages({
   emoji: { id: 'emoji_button.label', defaultMessage: 'Insert emoji' },
@@ -31,19 +31,6 @@ let EmojiPicker, Emoji; // load asynchronously
 const backgroundImageFn = () => `${assetHost}/emoji/sheet_10.png`;
 const listenerOptions = detectPassiveEvents.hasSupport ? { passive: true } : false;
 
-const categoriesSort = [
-  'recent',
-  'custom',
-  'people',
-  'nature',
-  'foods',
-  'activity',
-  'places',
-  'objects',
-  'symbols',
-  'flags',
-];
-
 class ModifierPickerMenu extends React.PureComponent {
 
   static propTypes = {
@@ -241,8 +228,23 @@ class EmojiPickerMenu extends React.PureComponent {
     }
 
     const title = intl.formatMessage(messages.emoji);
+
     const { modifierOpen } = this.state;
 
+    const categoriesSort = [
+      'recent',
+      'people',
+      'nature',
+      'foods',
+      'activity',
+      'places',
+      'objects',
+      'symbols',
+      'flags',
+    ];
+
+    categoriesSort.splice(1, 0, ...Array.from(categoriesFromEmojis(custom_emojis)).sort());
+
     return (
       <div className={classNames('emoji-picker-dropdown__menu', { selecting: modifierOpen })} style={style} ref={this.setRef}>
         <EmojiPicker
diff --git a/app/javascript/mastodon/features/emoji/emoji.js b/app/javascript/mastodon/features/emoji/emoji.js
index 988cea253..01b5a6664 100644
--- a/app/javascript/mastodon/features/emoji/emoji.js
+++ b/app/javascript/mastodon/features/emoji/emoji.js
@@ -92,8 +92,11 @@ export const buildCustomEmojis = (customEmojis) => {
       keywords: [name],
       imageUrl: url,
       custom: true,
+      customCategory: emoji.get('category'),
     });
   });
 
   return emojis;
 };
+
+export const categoriesFromEmojis = customEmojis => customEmojis.reduce((set, emoji) => set.add(emoji.get('category') ? `custom-${emoji.get('category')}` : 'custom'), new Set());
diff --git a/app/models/custom_emoji.rb b/app/models/custom_emoji.rb
index e73cd9bd2..514cf4845 100644
--- a/app/models/custom_emoji.rb
+++ b/app/models/custom_emoji.rb
@@ -16,6 +16,7 @@
 #  uri                :string
 #  image_remote_url   :string
 #  visible_in_picker  :boolean          default(TRUE), not null
+#  category_id        :bigint(8)
 #
 
 class CustomEmoji < ApplicationRecord
@@ -27,6 +28,7 @@ class CustomEmoji < ApplicationRecord
     :(#{SHORTCODE_RE_FRAGMENT}):
     (?=[^[:alnum:]:]|$)/x
 
+  belongs_to :category, class_name: 'CustomEmojiCategory', optional: true
   has_one :local_counterpart, -> { where(domain: nil) }, class_name: 'CustomEmoji', primary_key: :shortcode, foreign_key: :shortcode
 
   has_attached_file :image, styles: { static: { format: 'png', convert_options: '-coalesce -strip' } }
diff --git a/app/models/custom_emoji_category.rb b/app/models/custom_emoji_category.rb
new file mode 100644
index 000000000..7d8c0ee2d
--- /dev/null
+++ b/app/models/custom_emoji_category.rb
@@ -0,0 +1,15 @@
+# frozen_string_literal: true
+
+# == Schema Information
+#
+# Table name: custom_emoji_categories
+#
+#  id         :bigint(8)        not null, primary key
+#  name       :string
+#  created_at :datetime         not null
+#  updated_at :datetime         not null
+#
+
+class CustomEmojiCategory < ApplicationRecord
+  has_many :emojis, class_name: 'CustomEmoji', foreign_key: 'category_id', inverse_of: :category
+end
diff --git a/app/serializers/rest/custom_emoji_serializer.rb b/app/serializers/rest/custom_emoji_serializer.rb
index 65686a866..aff58e4d9 100644
--- a/app/serializers/rest/custom_emoji_serializer.rb
+++ b/app/serializers/rest/custom_emoji_serializer.rb
@@ -5,6 +5,8 @@ class REST::CustomEmojiSerializer < ActiveModel::Serializer
 
   attributes :shortcode, :url, :static_url, :visible_in_picker
 
+  attribute :category, if: :category_loaded?
+
   def url
     full_asset_url(object.image.url)
   end
@@ -12,4 +14,12 @@ class REST::CustomEmojiSerializer < ActiveModel::Serializer
   def static_url
     full_asset_url(object.image.url(:static))
   end
+
+  def category
+    object.category.name
+  end
+
+  def category_loaded?
+    object.association(:category).loaded? && object.category.present?
+  end
 end
diff --git a/db/migrate/20190627222225_create_custom_emoji_categories.rb b/db/migrate/20190627222225_create_custom_emoji_categories.rb
new file mode 100644
index 000000000..4713793e6
--- /dev/null
+++ b/db/migrate/20190627222225_create_custom_emoji_categories.rb
@@ -0,0 +1,9 @@
+class CreateCustomEmojiCategories < ActiveRecord::Migration[5.2]
+  def change
+    create_table :custom_emoji_categories do |t|
+      t.string :name, index: { unique: true }
+
+      t.timestamps
+    end
+  end
+end
diff --git a/db/migrate/20190627222826_add_category_id_to_custom_emojis.rb b/db/migrate/20190627222826_add_category_id_to_custom_emojis.rb
new file mode 100644
index 000000000..873b4d05f
--- /dev/null
+++ b/db/migrate/20190627222826_add_category_id_to_custom_emojis.rb
@@ -0,0 +1,5 @@
+class AddCategoryIdToCustomEmojis < ActiveRecord::Migration[5.2]
+  def change
+    add_column :custom_emojis, :category_id, :bigint
+  end
+end
diff --git a/db/schema.rb b/db/schema.rb
index f633f4e3f..09e6c9fae 100644
--- a/db/schema.rb
+++ b/db/schema.rb
@@ -10,7 +10,7 @@
 #
 # It's strongly recommended that you check this file into your version control system.
 
-ActiveRecord::Schema.define(version: 2019_05_29_143559) do
+ActiveRecord::Schema.define(version: 2019_06_27_222826) do
 
   # These are extensions that must be enabled in order to support this database
   enable_extension "plpgsql"
@@ -208,6 +208,13 @@ ActiveRecord::Schema.define(version: 2019_05_29_143559) do
     t.index ["uri"], name: "index_conversations_on_uri", unique: true
   end
 
+  create_table "custom_emoji_categories", force: :cascade do |t|
+    t.string "name"
+    t.datetime "created_at", null: false
+    t.datetime "updated_at", null: false
+    t.index ["name"], name: "index_custom_emoji_categories_on_name", unique: true
+  end
+
   create_table "custom_emojis", force: :cascade do |t|
     t.string "shortcode", default: "", null: false
     t.string "domain"
@@ -221,6 +228,7 @@ ActiveRecord::Schema.define(version: 2019_05_29_143559) do
     t.string "uri"
     t.string "image_remote_url"
     t.boolean "visible_in_picker", default: true, null: false
+    t.bigint "category_id"
     t.index ["shortcode", "domain"], name: "index_custom_emojis_on_shortcode_and_domain", unique: true
   end
 
diff --git a/lib/mastodon/emoji_cli.rb b/lib/mastodon/emoji_cli.rb
index 97a822e45..beac1b1fd 100644
--- a/lib/mastodon/emoji_cli.rb
+++ b/lib/mastodon/emoji_cli.rb
@@ -15,6 +15,7 @@ module Mastodon
     option :suffix
     option :overwrite, type: :boolean
     option :unlisted, type: :boolean
+    option :category
     desc 'import PATH', 'Import emoji from a TAR GZIP archive at PATH'
     long_desc <<-LONG_DESC
       Imports custom emoji from a TAR GZIP archive specified by PATH.
@@ -22,6 +23,9 @@ module Mastodon
       Existing emoji will be skipped unless the --overwrite option
       is provided, in which case they will be overwritten.
 
+      You can specifiy a --category under which the emojis will be
+      grouped together.
+
       With the --prefix option, a prefix can be added to all
       generated shortcodes. Likewise, the --suffix option controls
       the suffix of all shortcodes.
@@ -33,6 +37,7 @@ module Mastodon
       imported = 0
       skipped  = 0
       failed   = 0
+      category = options[:category] ? CustomEmojiCategory.find_or_create_by(name: options[:category]) : nil
 
       Gem::Package::TarReader.new(Zlib::GzipReader.open(path)) do |tar|
         tar.each do |entry|
@@ -50,6 +55,7 @@ module Mastodon
           custom_emoji.image = StringIO.new(entry.read)
           custom_emoji.image_file_name = File.basename(entry.full_name)
           custom_emoji.visible_in_picker = !options[:unlisted]
+          custom_emoji.category = category
 
           if custom_emoji.save
             imported += 1
diff --git a/spec/fabricators/custom_emoji_category_fabricator.rb b/spec/fabricators/custom_emoji_category_fabricator.rb
new file mode 100644
index 000000000..f593b95ed
--- /dev/null
+++ b/spec/fabricators/custom_emoji_category_fabricator.rb
@@ -0,0 +1,3 @@
+Fabricator(:custom_emoji_category) do
+  name "MyString"
+end
diff --git a/spec/models/custom_emoji_category_spec.rb b/spec/models/custom_emoji_category_spec.rb
new file mode 100644
index 000000000..160033f4d
--- /dev/null
+++ b/spec/models/custom_emoji_category_spec.rb
@@ -0,0 +1,5 @@
+require 'rails_helper'
+
+RSpec.describe CustomEmojiCategory, type: :model do
+  pending "add some examples to (or delete) #{__FILE__}"
+end
diff --git a/yarn.lock b/yarn.lock
index 560d84e31..2c402782c 100644
--- a/yarn.lock
+++ b/yarn.lock
@@ -3384,8 +3384,8 @@ elliptic@^6.0.0:
     minimalistic-crypto-utils "^1.0.0"
 
 emoji-mart@Gargron/emoji-mart#build:
-  version "2.6.2"
-  resolved "https://codeload.github.com/Gargron/emoji-mart/tar.gz/ff00dc470b5b2d9f145a6d6e977a54de5df2b4c9"
+  version "2.6.3"
+  resolved "https://codeload.github.com/Gargron/emoji-mart/tar.gz/934f314fd8322276765066e8a2a6be5bac61b1cf"
 
 emoji-regex@^7.0.1, emoji-regex@^7.0.2:
   version "7.0.3"
-- 
cgit 


From 06424786a140e03fcb899232c3c418b7eaf26d5e Mon Sep 17 00:00:00 2001
From: Eugen Rochko <eugen@zeonfederated.com>
Date: Sun, 30 Jun 2019 16:10:43 +0200
Subject: Fix support for MP4 files that are actually M4V files (#11210)

Resolve #11187
---
 app/models/concerns/attachmentable.rb | 4 +++-
 1 file changed, 3 insertions(+), 1 deletion(-)

(limited to 'app/models')

diff --git a/app/models/concerns/attachmentable.rb b/app/models/concerns/attachmentable.rb
index 24f5968de..7c78bb456 100644
--- a/app/models/concerns/attachmentable.rb
+++ b/app/models/concerns/attachmentable.rb
@@ -60,7 +60,9 @@ module Attachmentable
   end
 
   def calculated_content_type(attachment)
-    Paperclip.run('file', '-b --mime :file', file: attachment.queued_for_write[:original].path).split(/[:;\s]+/).first.chomp
+    content_type = Paperclip.run('file', '-b --mime :file', file: attachment.queued_for_write[:original].path).split(/[:;\s]+/).first.chomp
+    content_type = 'video/mp4' if content_type == 'video/x-m4v'
+    content_type
   rescue Terrapin::CommandLineError
     ''
   end
-- 
cgit 


From 082b3b282f7863e4b10e9cb34624b4b167b75588 Mon Sep 17 00:00:00 2001
From: ThibG <thib@sitedethib.com>
Date: Sun, 30 Jun 2019 16:11:21 +0200
Subject: Fix expiration date of filters being set to “Never” when editing them
 (#11204)
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

When editing a custom filter, select the shortest preset duration that
still covers the remaining time of that filter.

Fixes #9506
---
 app/models/custom_filter.rb | 7 +++++++
 1 file changed, 7 insertions(+)

(limited to 'app/models')

diff --git a/app/models/custom_filter.rb b/app/models/custom_filter.rb
index 342207a55..382562fb8 100644
--- a/app/models/custom_filter.rb
+++ b/app/models/custom_filter.rb
@@ -35,6 +35,13 @@ class CustomFilter < ApplicationRecord
   before_validation :clean_up_contexts
   after_commit :remove_cache
 
+  def expires_in
+    return @expires_in if defined?(@expires_in)
+    return nil if expires_at.nil?
+
+    [30.minutes, 1.hour, 6.hours, 12.hours, 1.day, 1.week].find { |expires_in| expires_in.from_now >= expires_at }
+  end
+
   private
 
   def clean_up_contexts
-- 
cgit 


From 3fd6ab99e69e736e10c7022f07dfa34b60a4e54d Mon Sep 17 00:00:00 2001
From: Eugen Rochko <eugen@zeonfederated.com>
Date: Fri, 5 Jul 2019 02:14:56 +0200
Subject: Remove deprecated REST API `GET /api/v1/timelines/direct` (#11212)

---
 .../api/v1/timelines/direct_controller.rb          | 63 ----------------------
 app/models/status.rb                               | 41 --------------
 config/routes.rb                                   |  1 -
 .../api/v1/timelines/direct_controller_spec.rb     | 17 ------
 spec/models/status_spec.rb                         | 50 -----------------
 5 files changed, 172 deletions(-)
 delete mode 100644 app/controllers/api/v1/timelines/direct_controller.rb
 delete mode 100644 spec/controllers/api/v1/timelines/direct_controller_spec.rb

(limited to 'app/models')

diff --git a/app/controllers/api/v1/timelines/direct_controller.rb b/app/controllers/api/v1/timelines/direct_controller.rb
deleted file mode 100644
index d8a76d153..000000000
--- a/app/controllers/api/v1/timelines/direct_controller.rb
+++ /dev/null
@@ -1,63 +0,0 @@
-# frozen_string_literal: true
-
-class Api::V1::Timelines::DirectController < Api::BaseController
-  before_action -> { doorkeeper_authorize! :read, :'read:statuses' }, only: [:show]
-  before_action :require_user!, only: [:show]
-  after_action :insert_pagination_headers, unless: -> { @statuses.empty? }
-
-  respond_to :json
-
-  def show
-    @statuses = load_statuses
-    render json: @statuses, each_serializer: REST::StatusSerializer, relationships: StatusRelationshipsPresenter.new(@statuses, current_user&.account_id)
-  end
-
-  private
-
-  def load_statuses
-    cached_direct_statuses
-  end
-
-  def cached_direct_statuses
-    cache_collection direct_statuses, Status
-  end
-
-  def direct_statuses
-    direct_timeline_statuses
-  end
-
-  def direct_timeline_statuses
-    # this query requires built in pagination.
-    Status.as_direct_timeline(
-      current_account,
-      limit_param(DEFAULT_STATUSES_LIMIT),
-      params[:max_id],
-      params[:since_id],
-      true # returns array of cache_ids object
-    )
-  end
-
-  def insert_pagination_headers
-    set_pagination_headers(next_path, prev_path)
-  end
-
-  def pagination_params(core_params)
-    params.permit(:local, :limit).merge(core_params)
-  end
-
-  def next_path
-    api_v1_timelines_direct_url pagination_params(max_id: pagination_max_id)
-  end
-
-  def prev_path
-    api_v1_timelines_direct_url pagination_params(since_id: pagination_since_id)
-  end
-
-  def pagination_max_id
-    @statuses.last.id
-  end
-
-  def pagination_since_id
-    @statuses.first.id
-  end
-end
diff --git a/app/models/status.rb b/app/models/status.rb
index fb9bbc9a9..2258e2d07 100644
--- a/app/models/status.rb
+++ b/app/models/status.rb
@@ -280,47 +280,6 @@ class Status < ApplicationRecord
       where(account: [account] + account.following).where(visibility: [:public, :unlisted, :private])
     end
 
-    def as_direct_timeline(account, limit = 20, max_id = nil, since_id = nil, cache_ids = false)
-      # direct timeline is mix of direct message from_me and to_me.
-      # 2 queries are executed with pagination.
-      # constant expression using arel_table is required for partial index
-
-      # _from_me part does not require any timeline filters
-      query_from_me = where(account_id: account.id)
-                      .where(Status.arel_table[:visibility].eq(3))
-                      .limit(limit)
-                      .order('statuses.id DESC')
-
-      # _to_me part requires mute and block filter.
-      # FIXME: may we check mutes.hide_notifications?
-      query_to_me = Status
-                    .joins(:mentions)
-                    .merge(Mention.where(account_id: account.id))
-                    .where(Status.arel_table[:visibility].eq(3))
-                    .limit(limit)
-                    .order('mentions.status_id DESC')
-                    .not_excluded_by_account(account)
-
-      if max_id.present?
-        query_from_me = query_from_me.where('statuses.id < ?', max_id)
-        query_to_me = query_to_me.where('mentions.status_id < ?', max_id)
-      end
-
-      if since_id.present?
-        query_from_me = query_from_me.where('statuses.id > ?', since_id)
-        query_to_me = query_to_me.where('mentions.status_id > ?', since_id)
-      end
-
-      if cache_ids
-        # returns array of cache_ids object that have id and updated_at
-        (query_from_me.cache_ids.to_a + query_to_me.cache_ids.to_a).uniq(&:id).sort_by(&:id).reverse.take(limit)
-      else
-        # returns ActiveRecord.Relation
-        items = (query_from_me.select(:id).to_a + query_to_me.select(:id).to_a).uniq(&:id).sort_by(&:id).reverse.take(limit)
-        Status.where(id: items.map(&:id))
-      end
-    end
-
     def as_public_timeline(account = nil, local_only = false)
       query = timeline_scope(local_only).without_replies
 
diff --git a/config/routes.rb b/config/routes.rb
index 764db8db2..29db54a32 100644
--- a/config/routes.rb
+++ b/config/routes.rb
@@ -299,7 +299,6 @@ Rails.application.routes.draw do
       end
 
       namespace :timelines do
-        resource :direct, only: :show, controller: :direct
         resource :home, only: :show, controller: :home
         resource :public, only: :show, controller: :public
         resources :tag, only: :show
diff --git a/spec/controllers/api/v1/timelines/direct_controller_spec.rb b/spec/controllers/api/v1/timelines/direct_controller_spec.rb
deleted file mode 100644
index a22c2cbea..000000000
--- a/spec/controllers/api/v1/timelines/direct_controller_spec.rb
+++ /dev/null
@@ -1,17 +0,0 @@
-# frozen_string_literal: true
-
-require 'rails_helper'
-
-RSpec.describe Api::V1::Timelines::DirectController, type: :controller do
-  let(:user)  { Fabricate(:user) }
-  let(:token) { Fabricate(:accessible_access_token, resource_owner_id: user.id, scopes: 'read:statuses') }
-
-  describe 'GET #show' do
-    it 'returns 200' do
-      allow(controller).to receive(:doorkeeper_token) { token }
-      get :show
-
-      expect(response).to have_http_status(200)
-    end
-  end
-end
diff --git a/spec/models/status_spec.rb b/spec/models/status_spec.rb
index e8cf18af9..a8c567414 100644
--- a/spec/models/status_spec.rb
+++ b/spec/models/status_spec.rb
@@ -339,56 +339,6 @@ RSpec.describe Status, type: :model do
     end
   end
 
-  describe '.as_direct_timeline' do
-    let(:account) { Fabricate(:account) }
-    let(:followed) { Fabricate(:account) }
-    let(:not_followed) { Fabricate(:account) }
-
-    before do
-      Fabricate(:follow, account: account, target_account: followed)
-
-      @self_public_status = Fabricate(:status, account: account, visibility: :public)
-      @self_direct_status = Fabricate(:status, account: account, visibility: :direct)
-      @followed_public_status = Fabricate(:status, account: followed, visibility: :public)
-      @followed_direct_status = Fabricate(:status, account: followed, visibility: :direct)
-      @not_followed_direct_status = Fabricate(:status, account: not_followed, visibility: :direct)
-
-      @results = Status.as_direct_timeline(account)
-    end
-
-    it 'does not include public statuses from self' do
-      expect(@results).to_not include(@self_public_status)
-    end
-
-    it 'includes direct statuses from self' do
-      expect(@results).to include(@self_direct_status)
-    end
-
-    it 'does not include public statuses from followed' do
-      expect(@results).to_not include(@followed_public_status)
-    end
-
-    it 'does not include direct statuses not mentioning recipient from followed' do
-      expect(@results).to_not include(@followed_direct_status)
-    end
-
-    it 'does not include direct statuses not mentioning recipient from non-followed' do
-      expect(@results).to_not include(@not_followed_direct_status)
-    end
-
-    it 'includes direct statuses mentioning recipient from followed' do
-      Fabricate(:mention, account: account, status: @followed_direct_status)
-      results2 = Status.as_direct_timeline(account)
-      expect(results2).to include(@followed_direct_status)
-    end
-
-    it 'includes direct statuses mentioning recipient from non-followed' do
-      Fabricate(:mention, account: account, status: @not_followed_direct_status)
-      results2 = Status.as_direct_timeline(account)
-      expect(results2).to include(@not_followed_direct_status)
-    end
-  end
-
   describe '.as_public_timeline' do
     it 'only includes statuses with public visibility' do
       public_status = Fabricate(:status, visibility: :public)
-- 
cgit 


From ae003d54f956fa2cbb422b04519b3a3e602652c3 Mon Sep 17 00:00:00 2001
From: Eugen Rochko <eugen@zeonfederated.com>
Date: Sat, 6 Jul 2019 13:54:32 +0200
Subject: Fix option to send e-mail notification about account action always
 being true (#11242)

---
 app/models/admin/account_action.rb | 9 ++++++---
 1 file changed, 6 insertions(+), 3 deletions(-)

(limited to 'app/models')

diff --git a/app/models/admin/account_action.rb b/app/models/admin/account_action.rb
index 84c3f880d..bdbd342fb 100644
--- a/app/models/admin/account_action.rb
+++ b/app/models/admin/account_action.rb
@@ -17,10 +17,13 @@ class Admin::AccountAction
                 :type,
                 :text,
                 :report_id,
-                :warning_preset_id,
-                :send_email_notification
+                :warning_preset_id
 
-  attr_reader :warning
+  attr_reader :warning, :send_email_notification
+
+  def send_email_notification=(value)
+    @send_email_notification = ActiveModel::Type::Boolean.new.cast(value)
+  end
 
   def save!
     ApplicationRecord.transaction do
-- 
cgit 


From 23aeef52cc4540b4514e9f3b935b21f0530a3746 Mon Sep 17 00:00:00 2001
From: Eugen Rochko <eugen@zeonfederated.com>
Date: Sat, 6 Jul 2019 23:26:16 +0200
Subject: Remove Salmon and PubSubHubbub (#11205)

* Remove Salmon and PubSubHubbub endpoints

* Add error when trying to follow OStatus accounts

* Fix new accounts not being created in ResolveAccountService
---
 app/controllers/activitypub/inboxes_controller.rb  |   1 -
 app/controllers/admin/accounts_controller.rb       |  16 +-
 app/controllers/api/push_controller.rb             |  73 ------
 app/controllers/api/salmon_controller.rb           |  37 ---
 app/controllers/api/subscriptions_controller.rb    |  51 -----
 app/controllers/api/v1/follows_controller.rb       |  31 ---
 app/lib/ostatus/activity/base.rb                   |  71 ------
 app/lib/ostatus/activity/creation.rb               | 219 ------------------
 app/lib/ostatus/activity/deletion.rb               |  16 --
 app/lib/ostatus/activity/general.rb                |  20 --
 app/lib/ostatus/activity/post.rb                   |  23 --
 app/lib/ostatus/activity/remote.rb                 |  11 -
 app/lib/ostatus/activity/share.rb                  |  26 ---
 app/lib/ostatus/atom_serializer.rb                 |   2 -
 app/models/account.rb                              |   3 +-
 app/serializers/webfinger_serializer.rb            |   1 -
 app/services/authorize_follow_service.rb           |  12 +-
 app/services/batched_remove_status_service.rb      |  34 +--
 app/services/block_domain_service.rb               |   1 -
 app/services/block_service.rb                      |  12 +-
 app/services/concerns/author_extractor.rb          |  23 --
 app/services/concerns/stream_entry_renderer.rb     |   7 -
 app/services/favourite_service.rb                  |   6 -
 app/services/fetch_remote_account_service.rb       |  28 ---
 app/services/fetch_remote_status_service.rb        |  28 ---
 app/services/follow_service.rb                     |  24 +-
 app/services/post_status_service.rb                |   1 -
 app/services/process_feed_service.rb               |  31 ---
 app/services/process_interaction_service.rb        | 151 ------------
 app/services/process_mentions_service.rb           |   7 -
 app/services/pubsubhubbub/subscribe_service.rb     |  53 -----
 app/services/pubsubhubbub/unsubscribe_service.rb   |  31 ---
 app/services/reblog_service.rb                     |   4 -
 app/services/reject_follow_service.rb              |  12 +-
 app/services/remove_status_service.rb              |  13 --
 app/services/resolve_account_service.rb            | 131 +----------
 app/services/send_interaction_service.rb           |  39 ----
 app/services/subscribe_service.rb                  |  58 -----
 app/services/unblock_service.rb                    |  12 +-
 app/services/unfavourite_service.rb                |  13 +-
 app/services/unfollow_service.rb                   |  16 +-
 app/services/unsubscribe_service.rb                |  36 ---
 app/services/update_remote_profile_service.rb      |  66 ------
 app/services/verify_salmon_service.rb              |  26 ---
 app/views/accounts/show.html.haml                  |   1 -
 .../admin/subscriptions/_subscription.html.haml    |  18 --
 app/views/admin/subscriptions/index.html.haml      |  16 --
 app/views/well_known/webfinger/show.xml.ruby       |   5 -
 app/workers/after_remote_follow_request_worker.rb  |  24 +-
 app/workers/after_remote_follow_worker.rb          |  24 +-
 app/workers/notification_worker.rb                 |   4 +-
 app/workers/processing_worker.rb                   |   4 +-
 app/workers/pubsubhubbub/confirmation_worker.rb    |  75 +-----
 app/workers/pubsubhubbub/delivery_worker.rb        |  74 +-----
 app/workers/pubsubhubbub/distribution_worker.rb    |  25 +-
 .../pubsubhubbub/raw_distribution_worker.rb        |  15 +-
 app/workers/pubsubhubbub/subscribe_worker.rb       |  27 +--
 app/workers/pubsubhubbub/unsubscribe_worker.rb     |   8 +-
 app/workers/remote_profile_update_worker.rb        |   6 +-
 app/workers/salmon_worker.rb                       |   6 +-
 app/workers/scheduler/subscriptions_scheduler.rb   |  10 +-
 config/locales/en.yml                              |   7 -
 config/navigation.rb                               |   1 -
 config/routes.rb                                   |  14 --
 config/sidekiq.yml                                 |   3 -
 spec/controllers/admin/accounts_controller_spec.rb |  38 ----
 .../admin/subscriptions_controller_spec.rb         |  32 ---
 spec/controllers/api/push_controller_spec.rb       |  59 -----
 spec/controllers/api/salmon_controller_spec.rb     |  65 ------
 .../api/subscriptions_controller_spec.rb           |  68 ------
 spec/controllers/api/v1/follows_controller_spec.rb |  51 -----
 spec/fixtures/requests/webfinger.txt               |   2 +-
 spec/lib/ostatus/atom_serializer_spec.rb           | 145 ------------
 spec/services/authorize_follow_service_spec.rb     |   7 -
 .../services/batched_remove_status_service_spec.rb |  13 --
 spec/services/block_service_spec.rb                |   7 -
 spec/services/favourite_service_spec.rb            |   7 -
 spec/services/fetch_remote_account_service_spec.rb |  40 ----
 spec/services/follow_service_spec.rb               |  68 ------
 spec/services/import_service_spec.rb               |  30 ++-
 spec/services/post_status_service_spec.rb          |   2 -
 spec/services/process_feed_service_spec.rb         | 252 ---------------------
 spec/services/process_interaction_service_spec.rb  | 151 ------------
 spec/services/process_mentions_service_spec.rb     |   4 -
 .../pubsubhubbub/subscribe_service_spec.rb         |  71 ------
 .../pubsubhubbub/unsubscribe_service_spec.rb       |  46 ----
 spec/services/reblog_service_spec.rb               |   4 -
 spec/services/reject_follow_service_spec.rb        |   7 -
 spec/services/remove_status_service_spec.rb        |  13 --
 spec/services/resolve_account_service_spec.rb      |  88 +------
 spec/services/send_interaction_service_spec.rb     |   7 -
 spec/services/subscribe_service_spec.rb            |  43 ----
 spec/services/unblock_service_spec.rb              |   7 -
 spec/services/unfollow_service_spec.rb             |   7 -
 spec/services/unsubscribe_service_spec.rb          |  37 ---
 .../services/update_remote_profile_service_spec.rb |  84 -------
 .../after_remote_follow_request_worker_spec.rb     |  59 -----
 spec/workers/after_remote_follow_worker_spec.rb    |  59 -----
 .../pubsubhubbub/confirmation_worker_spec.rb       |  88 -------
 spec/workers/pubsubhubbub/delivery_worker_spec.rb  |  68 ------
 .../pubsubhubbub/distribution_worker_spec.rb       |  46 ----
 .../scheduler/subscriptions_scheduler_spec.rb      |  19 --
 102 files changed, 69 insertions(+), 3568 deletions(-)
 delete mode 100644 app/controllers/api/push_controller.rb
 delete mode 100644 app/controllers/api/salmon_controller.rb
 delete mode 100644 app/controllers/api/subscriptions_controller.rb
 delete mode 100644 app/controllers/api/v1/follows_controller.rb
 delete mode 100644 app/lib/ostatus/activity/base.rb
 delete mode 100644 app/lib/ostatus/activity/creation.rb
 delete mode 100644 app/lib/ostatus/activity/deletion.rb
 delete mode 100644 app/lib/ostatus/activity/general.rb
 delete mode 100644 app/lib/ostatus/activity/post.rb
 delete mode 100644 app/lib/ostatus/activity/remote.rb
 delete mode 100644 app/lib/ostatus/activity/share.rb
 delete mode 100644 app/services/concerns/author_extractor.rb
 delete mode 100644 app/services/concerns/stream_entry_renderer.rb
 delete mode 100644 app/services/process_feed_service.rb
 delete mode 100644 app/services/process_interaction_service.rb
 delete mode 100644 app/services/pubsubhubbub/subscribe_service.rb
 delete mode 100644 app/services/pubsubhubbub/unsubscribe_service.rb
 delete mode 100644 app/services/send_interaction_service.rb
 delete mode 100644 app/services/subscribe_service.rb
 delete mode 100644 app/services/unsubscribe_service.rb
 delete mode 100644 app/services/update_remote_profile_service.rb
 delete mode 100644 app/services/verify_salmon_service.rb
 delete mode 100644 app/views/admin/subscriptions/_subscription.html.haml
 delete mode 100644 app/views/admin/subscriptions/index.html.haml
 delete mode 100644 spec/controllers/admin/subscriptions_controller_spec.rb
 delete mode 100644 spec/controllers/api/push_controller_spec.rb
 delete mode 100644 spec/controllers/api/salmon_controller_spec.rb
 delete mode 100644 spec/controllers/api/subscriptions_controller_spec.rb
 delete mode 100644 spec/controllers/api/v1/follows_controller_spec.rb
 delete mode 100644 spec/services/process_feed_service_spec.rb
 delete mode 100644 spec/services/process_interaction_service_spec.rb
 delete mode 100644 spec/services/pubsubhubbub/subscribe_service_spec.rb
 delete mode 100644 spec/services/pubsubhubbub/unsubscribe_service_spec.rb
 delete mode 100644 spec/services/send_interaction_service_spec.rb
 delete mode 100644 spec/services/subscribe_service_spec.rb
 delete mode 100644 spec/services/unsubscribe_service_spec.rb
 delete mode 100644 spec/services/update_remote_profile_service_spec.rb
 delete mode 100644 spec/workers/after_remote_follow_request_worker_spec.rb
 delete mode 100644 spec/workers/after_remote_follow_worker_spec.rb
 delete mode 100644 spec/workers/pubsubhubbub/confirmation_worker_spec.rb
 delete mode 100644 spec/workers/pubsubhubbub/delivery_worker_spec.rb
 delete mode 100644 spec/workers/pubsubhubbub/distribution_worker_spec.rb
 delete mode 100644 spec/workers/scheduler/subscriptions_scheduler_spec.rb

(limited to 'app/models')

diff --git a/app/controllers/activitypub/inboxes_controller.rb b/app/controllers/activitypub/inboxes_controller.rb
index a0b7532c2..e2cd8eaed 100644
--- a/app/controllers/activitypub/inboxes_controller.rb
+++ b/app/controllers/activitypub/inboxes_controller.rb
@@ -44,7 +44,6 @@ class ActivityPub::InboxesController < Api::BaseController
       ResolveAccountWorker.perform_async(signed_request_account.acct)
     end
 
-    Pubsubhubbub::UnsubscribeWorker.perform_async(signed_request_account.id) if signed_request_account.subscribed?
     DeliveryFailureTracker.track_inverse_success!(signed_request_account)
   end
 
diff --git a/app/controllers/admin/accounts_controller.rb b/app/controllers/admin/accounts_controller.rb
index 0c7760d77..2fa1dfe5f 100644
--- a/app/controllers/admin/accounts_controller.rb
+++ b/app/controllers/admin/accounts_controller.rb
@@ -2,8 +2,8 @@
 
 module Admin
   class AccountsController < BaseController
-    before_action :set_account, only: [:show, :subscribe, :unsubscribe, :redownload, :remove_avatar, :remove_header, :enable, :unsilence, :unsuspend, :memorialize, :approve, :reject]
-    before_action :require_remote_account!, only: [:subscribe, :unsubscribe, :redownload]
+    before_action :set_account, only: [:show, :redownload, :remove_avatar, :remove_header, :enable, :unsilence, :unsuspend, :memorialize, :approve, :reject]
+    before_action :require_remote_account!, only: [:redownload]
     before_action :require_local_account!, only: [:enable, :memorialize, :approve, :reject]
 
     def index
@@ -19,18 +19,6 @@ module Admin
       @warnings                = @account.targeted_account_warnings.latest.custom
     end
 
-    def subscribe
-      authorize @account, :subscribe?
-      Pubsubhubbub::SubscribeWorker.perform_async(@account.id)
-      redirect_to admin_account_path(@account.id)
-    end
-
-    def unsubscribe
-      authorize @account, :unsubscribe?
-      Pubsubhubbub::UnsubscribeWorker.perform_async(@account.id)
-      redirect_to admin_account_path(@account.id)
-    end
-
     def memorialize
       authorize @account, :memorialize?
       @account.memorialize!
diff --git a/app/controllers/api/push_controller.rb b/app/controllers/api/push_controller.rb
deleted file mode 100644
index e04d19125..000000000
--- a/app/controllers/api/push_controller.rb
+++ /dev/null
@@ -1,73 +0,0 @@
-# frozen_string_literal: true
-
-class Api::PushController < Api::BaseController
-  include SignatureVerification
-
-  def update
-    response, status = process_push_request
-    render plain: response, status: status
-  end
-
-  private
-
-  def process_push_request
-    case hub_mode
-    when 'subscribe'
-      Pubsubhubbub::SubscribeService.new.call(account_from_topic, hub_callback, hub_secret, hub_lease_seconds, verified_domain)
-    when 'unsubscribe'
-      Pubsubhubbub::UnsubscribeService.new.call(account_from_topic, hub_callback)
-    else
-      ["Unknown mode: #{hub_mode}", 422]
-    end
-  end
-
-  def hub_mode
-    params['hub.mode']
-  end
-
-  def hub_topic
-    params['hub.topic']
-  end
-
-  def hub_callback
-    params['hub.callback']
-  end
-
-  def hub_lease_seconds
-    params['hub.lease_seconds']
-  end
-
-  def hub_secret
-    params['hub.secret']
-  end
-
-  def account_from_topic
-    if hub_topic.present? && local_domain? && account_feed_path?
-      Account.find_local(hub_topic_params[:username])
-    end
-  end
-
-  def hub_topic_params
-    @_hub_topic_params ||= Rails.application.routes.recognize_path(hub_topic_uri.path)
-  end
-
-  def hub_topic_uri
-    @_hub_topic_uri ||= Addressable::URI.parse(hub_topic).normalize
-  end
-
-  def local_domain?
-    TagManager.instance.web_domain?(hub_topic_domain)
-  end
-
-  def verified_domain
-    return signed_request_account.domain if signed_request_account
-  end
-
-  def hub_topic_domain
-    hub_topic_uri.host + (hub_topic_uri.port ? ":#{hub_topic_uri.port}" : '')
-  end
-
-  def account_feed_path?
-    hub_topic_params[:controller] == 'accounts' && hub_topic_params[:action] == 'show' && hub_topic_params[:format] == 'atom'
-  end
-end
diff --git a/app/controllers/api/salmon_controller.rb b/app/controllers/api/salmon_controller.rb
deleted file mode 100644
index ac5f3268d..000000000
--- a/app/controllers/api/salmon_controller.rb
+++ /dev/null
@@ -1,37 +0,0 @@
-# frozen_string_literal: true
-
-class Api::SalmonController < Api::BaseController
-  include SignatureVerification
-
-  before_action :set_account
-  respond_to :txt
-
-  def update
-    if verify_payload?
-      process_salmon
-      head 202
-    elsif payload.present?
-      render plain: signature_verification_failure_reason, status: 401
-    else
-      head 400
-    end
-  end
-
-  private
-
-  def set_account
-    @account = Account.find(params[:id])
-  end
-
-  def payload
-    @_payload ||= request.body.read
-  end
-
-  def verify_payload?
-    payload.present? && VerifySalmonService.new.call(payload)
-  end
-
-  def process_salmon
-    SalmonWorker.perform_async(@account.id, payload.force_encoding('UTF-8'))
-  end
-end
diff --git a/app/controllers/api/subscriptions_controller.rb b/app/controllers/api/subscriptions_controller.rb
deleted file mode 100644
index 89007f3d6..000000000
--- a/app/controllers/api/subscriptions_controller.rb
+++ /dev/null
@@ -1,51 +0,0 @@
-# frozen_string_literal: true
-
-class Api::SubscriptionsController < Api::BaseController
-  before_action :set_account
-  respond_to :txt
-
-  def show
-    if subscription.valid?(params['hub.topic'])
-      @account.update(subscription_expires_at: future_expires)
-      render plain: encoded_challenge, status: 200
-    else
-      head 404
-    end
-  end
-
-  def update
-    if subscription.verify(body, request.headers['HTTP_X_HUB_SIGNATURE'])
-      ProcessingWorker.perform_async(@account.id, body.force_encoding('UTF-8'))
-    end
-
-    head 200
-  end
-
-  private
-
-  def subscription
-    @_subscription ||= @account.subscription(
-      api_subscription_url(@account.id)
-    )
-  end
-
-  def body
-    @_body ||= request.body.read
-  end
-
-  def encoded_challenge
-    HTMLEntities.new.encode(params['hub.challenge'])
-  end
-
-  def future_expires
-    Time.now.utc + lease_seconds_or_default
-  end
-
-  def lease_seconds_or_default
-    (params['hub.lease_seconds'] || 1.day).to_i.seconds
-  end
-
-  def set_account
-    @account = Account.find(params[:id])
-  end
-end
diff --git a/app/controllers/api/v1/follows_controller.rb b/app/controllers/api/v1/follows_controller.rb
deleted file mode 100644
index 5420c0533..000000000
--- a/app/controllers/api/v1/follows_controller.rb
+++ /dev/null
@@ -1,31 +0,0 @@
-# frozen_string_literal: true
-
-class Api::V1::FollowsController < Api::BaseController
-  before_action -> { doorkeeper_authorize! :follow, :'write:follows' }
-  before_action :require_user!
-
-  respond_to :json
-
-  def create
-    raise ActiveRecord::RecordNotFound if follow_params[:uri].blank?
-
-    @account = FollowService.new.call(current_user.account, target_uri).try(:target_account)
-
-    if @account.nil?
-      username, domain = target_uri.split('@')
-      @account         = Account.find_remote!(username, domain)
-    end
-
-    render json: @account, serializer: REST::AccountSerializer
-  end
-
-  private
-
-  def target_uri
-    follow_params[:uri].strip.gsub(/\A@/, '')
-  end
-
-  def follow_params
-    params.permit(:uri)
-  end
-end
diff --git a/app/lib/ostatus/activity/base.rb b/app/lib/ostatus/activity/base.rb
deleted file mode 100644
index db70f1998..000000000
--- a/app/lib/ostatus/activity/base.rb
+++ /dev/null
@@ -1,71 +0,0 @@
-# frozen_string_literal: true
-
-class OStatus::Activity::Base
-  include Redisable
-
-  def initialize(xml, account = nil, **options)
-    @xml     = xml
-    @account = account
-    @options = options
-  end
-
-  def status?
-    [:activity, :note, :comment].include?(type)
-  end
-
-  def verb
-    raw = @xml.at_xpath('./activity:verb', activity: OStatus::TagManager::AS_XMLNS).content
-    OStatus::TagManager::VERBS.key(raw)
-  rescue
-    :post
-  end
-
-  def type
-    raw = @xml.at_xpath('./activity:object-type', activity: OStatus::TagManager::AS_XMLNS).content
-    OStatus::TagManager::TYPES.key(raw)
-  rescue
-    :activity
-  end
-
-  def id
-    @xml.at_xpath('./xmlns:id', xmlns: OStatus::TagManager::XMLNS).content
-  end
-
-  def url
-    link = @xml.xpath('./xmlns:link[@rel="alternate"]', xmlns: OStatus::TagManager::XMLNS).find { |link_candidate| link_candidate['type'] == 'text/html' }
-    link.nil? ? nil : link['href']
-  end
-
-  def activitypub_uri
-    link = @xml.xpath('./xmlns:link[@rel="alternate"]', xmlns: OStatus::TagManager::XMLNS).find { |link_candidate| ['application/activity+json', 'application/ld+json; profile="https://www.w3.org/ns/activitystreams"'].include?(link_candidate['type']) }
-    link.nil? ? nil : link['href']
-  end
-
-  def activitypub_uri?
-    activitypub_uri.present?
-  end
-
-  private
-
-  def find_status(uri)
-    if OStatus::TagManager.instance.local_id?(uri)
-      local_id = OStatus::TagManager.instance.unique_tag_to_local_id(uri, 'Status')
-      return Status.find_by(id: local_id)
-    elsif ActivityPub::TagManager.instance.local_uri?(uri)
-      local_id = ActivityPub::TagManager.instance.uri_to_local_id(uri)
-      return Status.find_by(id: local_id)
-    end
-
-    Status.find_by(uri: uri)
-  end
-
-  def find_activitypub_status(uri, href)
-    tag_matches = /tag:([^,:]+)[^:]*:objectId=([\d]+)/.match(uri)
-    href_matches = %r{/users/([^/]+)}.match(href)
-
-    unless tag_matches.nil? || href_matches.nil?
-      uri = "https://#{tag_matches[1]}/users/#{href_matches[1]}/statuses/#{tag_matches[2]}"
-      Status.find_by(uri: uri)
-    end
-  end
-end
diff --git a/app/lib/ostatus/activity/creation.rb b/app/lib/ostatus/activity/creation.rb
deleted file mode 100644
index 60de712db..000000000
--- a/app/lib/ostatus/activity/creation.rb
+++ /dev/null
@@ -1,219 +0,0 @@
-# frozen_string_literal: true
-
-class OStatus::Activity::Creation < OStatus::Activity::Base
-  def perform
-    if redis.exists("delete_upon_arrival:#{@account.id}:#{id}")
-      Rails.logger.debug "Delete for status #{id} was queued, ignoring"
-      return [nil, false]
-    end
-
-    return [nil, false] if @account.suspended? || invalid_origin?
-
-    RedisLock.acquire(lock_options) do |lock|
-      if lock.acquired?
-        # Return early if status already exists in db
-        @status = find_status(id)
-        return [@status, false] unless @status.nil?
-        @status = process_status
-      else
-        raise Mastodon::RaceConditionError
-      end
-    end
-
-    [@status, true]
-  end
-
-  def process_status
-    Rails.logger.debug "Creating remote status #{id}"
-    cached_reblog = reblog
-    status = nil
-
-    # Skip if the reblogged status is not public
-    return if cached_reblog && !(cached_reblog.public_visibility? || cached_reblog.unlisted_visibility?)
-
-    media_attachments = save_media.take(4)
-
-    ApplicationRecord.transaction do
-      status = Status.create!(
-        uri: id,
-        url: url,
-        account: @account,
-        reblog: cached_reblog,
-        text: content,
-        spoiler_text: content_warning,
-        created_at: published,
-        override_timestamps: @options[:override_timestamps],
-        reply: thread?,
-        language: content_language,
-        visibility: visibility_scope,
-        conversation: find_or_create_conversation,
-        thread: thread? ? find_status(thread.first) || find_activitypub_status(thread.first, thread.second) : nil,
-        media_attachment_ids: media_attachments.map(&:id),
-        sensitive: sensitive?
-      )
-
-      save_mentions(status)
-      save_hashtags(status)
-      save_emojis(status)
-    end
-
-    if thread? && status.thread.nil? && Request.valid_url?(thread.second)
-      Rails.logger.debug "Trying to attach #{status.id} (#{id}) to #{thread.first}"
-      ThreadResolveWorker.perform_async(status.id, thread.second)
-    end
-
-    Rails.logger.debug "Queuing remote status #{status.id} (#{id}) for distribution"
-
-    LinkCrawlWorker.perform_async(status.id) unless status.spoiler_text?
-
-    # Only continue if the status is supposed to have arrived in real-time.
-    # Note that if @options[:override_timestamps] isn't set, the status
-    # may have a lower snowflake id than other existing statuses, potentially
-    # "hiding" it from paginated API calls
-    return status unless @options[:override_timestamps] || status.within_realtime_window?
-
-    DistributionWorker.perform_async(status.id)
-
-    status
-  end
-
-  def content
-    @xml.at_xpath('./xmlns:content', xmlns: OStatus::TagManager::XMLNS).content
-  end
-
-  def content_language
-    @xml.at_xpath('./xmlns:content', xmlns: OStatus::TagManager::XMLNS)['xml:lang']&.presence || 'en'
-  end
-
-  def content_warning
-    @xml.at_xpath('./xmlns:summary', xmlns: OStatus::TagManager::XMLNS)&.content || ''
-  end
-
-  def visibility_scope
-    @xml.at_xpath('./mastodon:scope', mastodon: OStatus::TagManager::MTDN_XMLNS)&.content&.to_sym || :public
-  end
-
-  def published
-    @xml.at_xpath('./xmlns:published', xmlns: OStatus::TagManager::XMLNS).content
-  end
-
-  def thread?
-    !@xml.at_xpath('./thr:in-reply-to', thr: OStatus::TagManager::THR_XMLNS).nil?
-  end
-
-  def thread
-    thr = @xml.at_xpath('./thr:in-reply-to', thr: OStatus::TagManager::THR_XMLNS)
-    [thr['ref'], thr['href']]
-  end
-
-  private
-
-  def sensitive?
-    # OStatus-specific convention (not standard)
-    @xml.xpath('./xmlns:category', xmlns: OStatus::TagManager::XMLNS).any? { |category| category['term'] == 'nsfw' }
-  end
-
-  def find_or_create_conversation
-    uri = @xml.at_xpath('./ostatus:conversation', ostatus: OStatus::TagManager::OS_XMLNS)&.attribute('ref')&.content
-    return if uri.nil?
-
-    if OStatus::TagManager.instance.local_id?(uri)
-      local_id = OStatus::TagManager.instance.unique_tag_to_local_id(uri, 'Conversation')
-      return Conversation.find_by(id: local_id)
-    end
-
-    Conversation.find_by(uri: uri) || Conversation.create!(uri: uri)
-  end
-
-  def save_mentions(parent)
-    processed_account_ids = []
-
-    @xml.xpath('./xmlns:link[@rel="mentioned"]', xmlns: OStatus::TagManager::XMLNS).each do |link|
-      next if [OStatus::TagManager::TYPES[:group], OStatus::TagManager::TYPES[:collection]].include? link['ostatus:object-type']
-
-      mentioned_account = account_from_href(link['href'])
-
-      next if mentioned_account.nil? || processed_account_ids.include?(mentioned_account.id)
-
-      mentioned_account.mentions.where(status: parent).first_or_create(status: parent)
-
-      # So we can skip duplicate mentions
-      processed_account_ids << mentioned_account.id
-    end
-  end
-
-  def save_hashtags(parent)
-    tags = @xml.xpath('./xmlns:category', xmlns: OStatus::TagManager::XMLNS).map { |category| category['term'] }.select(&:present?)
-    ProcessHashtagsService.new.call(parent, tags)
-  end
-
-  def save_media
-    do_not_download = DomainBlock.reject_media?(@account.domain)
-    media_attachments = []
-
-    @xml.xpath('./xmlns:link[@rel="enclosure"]', xmlns: OStatus::TagManager::XMLNS).each do |link|
-      next unless link['href']
-
-      media = MediaAttachment.where(status: nil, remote_url: link['href']).first_or_initialize(account: @account, status: nil, remote_url: link['href'])
-      parsed_url = Addressable::URI.parse(link['href']).normalize
-
-      next if !%w(http https).include?(parsed_url.scheme) || parsed_url.host.empty?
-
-      media.save
-      media_attachments << media
-
-      next if do_not_download
-
-      begin
-        media.file_remote_url = link['href']
-        media.save!
-      rescue ActiveRecord::RecordInvalid
-        next
-      end
-    end
-
-    media_attachments
-  end
-
-  def save_emojis(parent)
-    do_not_download = DomainBlock.reject_media?(parent.account.domain)
-
-    return if do_not_download
-
-    @xml.xpath('./xmlns:link[@rel="emoji"]', xmlns: OStatus::TagManager::XMLNS).each do |link|
-      next unless link['href'] && link['name']
-
-      shortcode = link['name'].delete(':')
-      emoji     = CustomEmoji.find_by(shortcode: shortcode, domain: parent.account.domain)
-
-      next unless emoji.nil?
-
-      emoji = CustomEmoji.new(shortcode: shortcode, domain: parent.account.domain)
-      emoji.image_remote_url = link['href']
-      emoji.save
-    end
-  end
-
-  def account_from_href(href)
-    url = Addressable::URI.parse(href).normalize
-
-    if TagManager.instance.web_domain?(url.host)
-      Account.find_local(url.path.gsub('/users/', ''))
-    else
-      Account.where(uri: href).or(Account.where(url: href)).first || FetchRemoteAccountService.new.call(href)
-    end
-  end
-
-  def invalid_origin?
-    return false unless id.start_with?('http') # Legacy IDs cannot be checked
-
-    needle = Addressable::URI.parse(id).normalized_host
-
-    !(needle.casecmp(@account.domain).zero? ||
-      needle.casecmp(Addressable::URI.parse(@account.remote_url.presence || @account.uri).normalized_host).zero?)
-  end
-
-  def lock_options
-    { redis: Redis.current, key: "create:#{id}" }
-  end
-end
diff --git a/app/lib/ostatus/activity/deletion.rb b/app/lib/ostatus/activity/deletion.rb
deleted file mode 100644
index c98f5ee0a..000000000
--- a/app/lib/ostatus/activity/deletion.rb
+++ /dev/null
@@ -1,16 +0,0 @@
-# frozen_string_literal: true
-
-class OStatus::Activity::Deletion < OStatus::Activity::Base
-  def perform
-    Rails.logger.debug "Deleting remote status #{id}"
-
-    status   = Status.find_by(uri: id, account: @account)
-    status ||= Status.find_by(uri: activitypub_uri, account: @account) if activitypub_uri?
-
-    if status.nil?
-      redis.setex("delete_upon_arrival:#{@account.id}:#{id}", 6 * 3_600, id)
-    else
-      RemoveStatusService.new.call(status)
-    end
-  end
-end
diff --git a/app/lib/ostatus/activity/general.rb b/app/lib/ostatus/activity/general.rb
deleted file mode 100644
index 8a6aabc33..000000000
--- a/app/lib/ostatus/activity/general.rb
+++ /dev/null
@@ -1,20 +0,0 @@
-# frozen_string_literal: true
-
-class OStatus::Activity::General < OStatus::Activity::Base
-  def specialize
-    special_class&.new(@xml, @account, @options)
-  end
-
-  private
-
-  def special_class
-    case verb
-    when :post
-      OStatus::Activity::Post
-    when :share
-      OStatus::Activity::Share
-    when :delete
-      OStatus::Activity::Deletion
-    end
-  end
-end
diff --git a/app/lib/ostatus/activity/post.rb b/app/lib/ostatus/activity/post.rb
deleted file mode 100644
index 755ed8656..000000000
--- a/app/lib/ostatus/activity/post.rb
+++ /dev/null
@@ -1,23 +0,0 @@
-# frozen_string_literal: true
-
-class OStatus::Activity::Post < OStatus::Activity::Creation
-  def perform
-    status, just_created = super
-
-    if just_created
-      status.mentions.includes(:account).each do |mention|
-        mentioned_account = mention.account
-        next unless mentioned_account.local?
-        NotifyService.new.call(mentioned_account, mention)
-      end
-    end
-
-    status
-  end
-
-  private
-
-  def reblog
-    nil
-  end
-end
diff --git a/app/lib/ostatus/activity/remote.rb b/app/lib/ostatus/activity/remote.rb
deleted file mode 100644
index 5b204b6d8..000000000
--- a/app/lib/ostatus/activity/remote.rb
+++ /dev/null
@@ -1,11 +0,0 @@
-# frozen_string_literal: true
-
-class OStatus::Activity::Remote < OStatus::Activity::Base
-  def perform
-    if activitypub_uri?
-      find_status(activitypub_uri) || FetchRemoteStatusService.new.call(url)
-    else
-      find_status(id) || FetchRemoteStatusService.new.call(url)
-    end
-  end
-end
diff --git a/app/lib/ostatus/activity/share.rb b/app/lib/ostatus/activity/share.rb
deleted file mode 100644
index 5ca601415..000000000
--- a/app/lib/ostatus/activity/share.rb
+++ /dev/null
@@ -1,26 +0,0 @@
-# frozen_string_literal: true
-
-class OStatus::Activity::Share < OStatus::Activity::Creation
-  def perform
-    return if reblog.nil?
-
-    status, just_created = super
-    NotifyService.new.call(reblog.account, status) if reblog.account.local? && just_created
-    status
-  end
-
-  def object
-    @xml.at_xpath('.//activity:object', activity: OStatus::TagManager::AS_XMLNS)
-  end
-
-  private
-
-  def reblog
-    return @reblog if defined? @reblog
-
-    original_status = OStatus::Activity::Remote.new(object).perform
-    return if original_status.nil?
-
-    @reblog = original_status.reblog? ? original_status.reblog : original_status
-  end
-end
diff --git a/app/lib/ostatus/atom_serializer.rb b/app/lib/ostatus/atom_serializer.rb
index 9a05d96cf..f5c0e85ca 100644
--- a/app/lib/ostatus/atom_serializer.rb
+++ b/app/lib/ostatus/atom_serializer.rb
@@ -53,8 +53,6 @@ class OStatus::AtomSerializer
     append_element(feed, 'link', nil, rel: :alternate, type: 'text/html', href: ::TagManager.instance.url_for(account))
     append_element(feed, 'link', nil, rel: :self, type: 'application/atom+xml', href: account_url(account, format: 'atom'))
     append_element(feed, 'link', nil, rel: :next, type: 'application/atom+xml', href: account_url(account, format: 'atom', max_id: stream_entries.last.id)) if stream_entries.size == 20
-    append_element(feed, 'link', nil, rel: :hub, href: api_push_url)
-    append_element(feed, 'link', nil, rel: :salmon, href: api_salmon_url(account.id))
 
     stream_entries.each do |stream_entry|
       feed << entry(stream_entry)
diff --git a/app/models/account.rb b/app/models/account.rb
index c588451fc..d6772eb98 100644
--- a/app/models/account.rb
+++ b/app/models/account.rb
@@ -164,8 +164,7 @@ class Account < ApplicationRecord
   end
 
   def refresh!
-    return if local?
-    ResolveAccountService.new.call(acct)
+    ResolveAccountService.new.call(acct) unless local?
   end
 
   def silenced?
diff --git a/app/serializers/webfinger_serializer.rb b/app/serializers/webfinger_serializer.rb
index 8c0b07702..4220f697e 100644
--- a/app/serializers/webfinger_serializer.rb
+++ b/app/serializers/webfinger_serializer.rb
@@ -18,7 +18,6 @@ class WebfingerSerializer < ActiveModel::Serializer
       { rel: 'http://webfinger.net/rel/profile-page', type: 'text/html', href: short_account_url(object) },
       { rel: 'http://schemas.google.com/g/2010#updates-from', type: 'application/atom+xml', href: account_url(object, format: 'atom') },
       { rel: 'self', type: 'application/activity+json', href: account_url(object) },
-      { rel: 'salmon', href: api_salmon_url(object.id) },
       { rel: 'magic-public-key', href: "data:application/magic-public-key,#{object.magic_key}" },
       { rel: 'http://ostatus.org/schema/1.0/subscribe', template: "#{authorize_interaction_url}?uri={uri}" },
     ]
diff --git a/app/services/authorize_follow_service.rb b/app/services/authorize_follow_service.rb
index 29b8700c7..49bef727e 100644
--- a/app/services/authorize_follow_service.rb
+++ b/app/services/authorize_follow_service.rb
@@ -11,25 +11,17 @@ class AuthorizeFollowService < BaseService
       follow_request.authorize!
     end
 
-    create_notification(follow_request) unless source_account.local?
+    create_notification(follow_request) if !source_account.local? && source_account.activitypub?
     follow_request
   end
 
   private
 
   def create_notification(follow_request)
-    if follow_request.account.ostatus?
-      NotificationWorker.perform_async(build_xml(follow_request), follow_request.target_account_id, follow_request.account_id)
-    elsif follow_request.account.activitypub?
-      ActivityPub::DeliveryWorker.perform_async(build_json(follow_request), follow_request.target_account_id, follow_request.account.inbox_url)
-    end
+    ActivityPub::DeliveryWorker.perform_async(build_json(follow_request), follow_request.target_account_id, follow_request.account.inbox_url)
   end
 
   def build_json(follow_request)
     Oj.dump(serialize_payload(follow_request, ActivityPub::AcceptFollowSerializer))
   end
-
-  def build_xml(follow_request)
-    OStatus::AtomSerializer.render(OStatus::AtomSerializer.new.authorize_follow_request_salmon(follow_request))
-  end
 end
diff --git a/app/services/batched_remove_status_service.rb b/app/services/batched_remove_status_service.rb
index e328b1739..cb66debc8 100644
--- a/app/services/batched_remove_status_service.rb
+++ b/app/services/batched_remove_status_service.rb
@@ -1,7 +1,6 @@
 # frozen_string_literal: true
 
 class BatchedRemoveStatusService < BaseService
-  include StreamEntryRenderer
   include Redisable
 
   # Delete given statuses and reblogs of them
@@ -18,10 +17,7 @@ class BatchedRemoveStatusService < BaseService
     @mentions = statuses.each_with_object({}) { |s, h| h[s.id] = s.active_mentions.includes(:account).to_a }
     @tags     = statuses.each_with_object({}) { |s, h| h[s.id] = s.tags.pluck(:name) }
 
-    @stream_entry_batches  = []
-    @salmon_batches        = []
-    @json_payloads         = statuses.each_with_object({}) { |s, h| h[s.id] = Oj.dump(event: :delete, payload: s.id.to_s) }
-    @activity_xml          = {}
+    @json_payloads = statuses.each_with_object({}) { |s, h| h[s.id] = Oj.dump(event: :delete, payload: s.id.to_s) }
 
     # Ensure that rendered XML reflects destroyed state
     statuses.each do |status|
@@ -39,28 +35,16 @@ class BatchedRemoveStatusService < BaseService
 
       unpush_from_home_timelines(account, account_statuses)
       unpush_from_list_timelines(account, account_statuses)
-
-      batch_stream_entries(account, account_statuses) if account.local?
     end
 
     # Cannot be batched
     statuses.each do |status|
       unpush_from_public_timelines(status)
-      batch_salmon_slaps(status) if status.local?
     end
-
-    Pubsubhubbub::RawDistributionWorker.push_bulk(@stream_entry_batches) { |batch| batch }
-    NotificationWorker.push_bulk(@salmon_batches) { |batch| batch }
   end
 
   private
 
-  def batch_stream_entries(account, statuses)
-    statuses.each do |status|
-      @stream_entry_batches << [build_xml(status.stream_entry), account.id]
-    end
-  end
-
   def unpush_from_home_timelines(account, statuses)
     recipients = account.followers_for_local_distribution.to_a
 
@@ -101,20 +85,4 @@ class BatchedRemoveStatusService < BaseService
       end
     end
   end
-
-  def batch_salmon_slaps(status)
-    return if @mentions[status.id].empty?
-
-    recipients = @mentions[status.id].map(&:account).reject(&:local?).select(&:ostatus?).uniq(&:domain).map(&:id)
-
-    recipients.each do |recipient_id|
-      @salmon_batches << [build_xml(status.stream_entry), status.account_id, recipient_id]
-    end
-  end
-
-  def build_xml(stream_entry)
-    return @activity_xml[stream_entry.id] if @activity_xml.key?(stream_entry.id)
-
-    @activity_xml[stream_entry.id] = stream_entry_to_xml(stream_entry)
-  end
 end
diff --git a/app/services/block_domain_service.rb b/app/services/block_domain_service.rb
index c6eef04d4..c5e5e5761 100644
--- a/app/services/block_domain_service.rb
+++ b/app/services/block_domain_service.rb
@@ -44,7 +44,6 @@ class BlockDomainService < BaseService
 
   def suspend_accounts!
     blocked_domain_accounts.without_suspended.reorder(nil).find_each do |account|
-      UnsubscribeService.new.call(account) if account.subscribed?
       SuspendAccountService.new.call(account, suspended_at: @domain_block.created_at)
     end
   end
diff --git a/app/services/block_service.rb b/app/services/block_service.rb
index 9050a4858..da06361c2 100644
--- a/app/services/block_service.rb
+++ b/app/services/block_service.rb
@@ -13,25 +13,17 @@ class BlockService < BaseService
     block = account.block!(target_account)
 
     BlockWorker.perform_async(account.id, target_account.id)
-    create_notification(block) unless target_account.local?
+    create_notification(block) if !target_account.local? && target_account.activitypub?
     block
   end
 
   private
 
   def create_notification(block)
-    if block.target_account.ostatus?
-      NotificationWorker.perform_async(build_xml(block), block.account_id, block.target_account_id)
-    elsif block.target_account.activitypub?
-      ActivityPub::DeliveryWorker.perform_async(build_json(block), block.account_id, block.target_account.inbox_url)
-    end
+    ActivityPub::DeliveryWorker.perform_async(build_json(block), block.account_id, block.target_account.inbox_url)
   end
 
   def build_json(block)
     Oj.dump(serialize_payload(block, ActivityPub::BlockSerializer))
   end
-
-  def build_xml(block)
-    OStatus::AtomSerializer.render(OStatus::AtomSerializer.new.block_salmon(block))
-  end
 end
diff --git a/app/services/concerns/author_extractor.rb b/app/services/concerns/author_extractor.rb
deleted file mode 100644
index c2419e9ec..000000000
--- a/app/services/concerns/author_extractor.rb
+++ /dev/null
@@ -1,23 +0,0 @@
-# frozen_string_literal: true
-
-module AuthorExtractor
-  def author_from_xml(xml, update_profile = true)
-    return nil if xml.nil?
-
-    # Try <email> for acct
-    acct = xml.at_xpath('./xmlns:author/xmlns:email', xmlns: OStatus::TagManager::XMLNS)&.content
-
-    # Try <name> + <uri>
-    if acct.blank?
-      username = xml.at_xpath('./xmlns:author/xmlns:name', xmlns: OStatus::TagManager::XMLNS)&.content
-      uri      = xml.at_xpath('./xmlns:author/xmlns:uri', xmlns: OStatus::TagManager::XMLNS)&.content
-
-      return nil if username.blank? || uri.blank?
-
-      domain = Addressable::URI.parse(uri).normalized_host
-      acct   = "#{username}@#{domain}"
-    end
-
-    ResolveAccountService.new.call(acct, update_profile: update_profile)
-  end
-end
diff --git a/app/services/concerns/stream_entry_renderer.rb b/app/services/concerns/stream_entry_renderer.rb
deleted file mode 100644
index 9f6c8a082..000000000
--- a/app/services/concerns/stream_entry_renderer.rb
+++ /dev/null
@@ -1,7 +0,0 @@
-# frozen_string_literal: true
-
-module StreamEntryRenderer
-  def stream_entry_to_xml(stream_entry)
-    OStatus::AtomSerializer.render(OStatus::AtomSerializer.new.entry(stream_entry, true))
-  end
-end
diff --git a/app/services/favourite_service.rb b/app/services/favourite_service.rb
index 128a24ad6..02b26458a 100644
--- a/app/services/favourite_service.rb
+++ b/app/services/favourite_service.rb
@@ -30,8 +30,6 @@ class FavouriteService < BaseService
 
     if status.account.local?
       NotifyService.new.call(status.account, favourite)
-    elsif status.account.ostatus?
-      NotificationWorker.perform_async(build_xml(favourite), favourite.account_id, status.account_id)
     elsif status.account.activitypub?
       ActivityPub::DeliveryWorker.perform_async(build_json(favourite), favourite.account_id, status.account.inbox_url)
     end
@@ -46,8 +44,4 @@ class FavouriteService < BaseService
   def build_json(favourite)
     Oj.dump(serialize_payload(favourite, ActivityPub::LikeSerializer))
   end
-
-  def build_xml(favourite)
-    OStatus::AtomSerializer.render(OStatus::AtomSerializer.new.favourite_salmon(favourite))
-  end
 end
diff --git a/app/services/fetch_remote_account_service.rb b/app/services/fetch_remote_account_service.rb
index cfc560022..a7f95603d 100644
--- a/app/services/fetch_remote_account_service.rb
+++ b/app/services/fetch_remote_account_service.rb
@@ -1,8 +1,6 @@
 # frozen_string_literal: true
 
 class FetchRemoteAccountService < BaseService
-  include AuthorExtractor
-
   def call(url, prefetched_body = nil, protocol = :ostatus)
     if prefetched_body.nil?
       resource_url, resource_options, protocol = FetchAtomService.new.call(url)
@@ -12,34 +10,8 @@ class FetchRemoteAccountService < BaseService
     end
 
     case protocol
-    when :ostatus
-      process_atom(resource_url, **resource_options)
     when :activitypub
       ActivityPub::FetchRemoteAccountService.new.call(resource_url, **resource_options)
     end
   end
-
-  private
-
-  def process_atom(url, prefetched_body:)
-    xml = Nokogiri::XML(prefetched_body)
-    xml.encoding = 'utf-8'
-
-    account = author_from_xml(xml.at_xpath('/xmlns:feed', xmlns: OStatus::TagManager::XMLNS), false)
-
-    UpdateRemoteProfileService.new.call(xml, account) if account.present? && trusted_domain?(url, account)
-
-    account
-  rescue TypeError
-    Rails.logger.debug "Unparseable URL given: #{url}"
-    nil
-  rescue Nokogiri::XML::XPath::SyntaxError
-    Rails.logger.debug 'Invalid XML or missing namespace'
-    nil
-  end
-
-  def trusted_domain?(url, account)
-    domain = Addressable::URI.parse(url).normalized_host
-    domain.casecmp(account.domain).zero? || domain.casecmp(Addressable::URI.parse(account.remote_url.presence || account.uri).normalized_host).zero?
-  end
 end
diff --git a/app/services/fetch_remote_status_service.rb b/app/services/fetch_remote_status_service.rb
index 9c3008035..aac39dfd5 100644
--- a/app/services/fetch_remote_status_service.rb
+++ b/app/services/fetch_remote_status_service.rb
@@ -1,8 +1,6 @@
 # frozen_string_literal: true
 
 class FetchRemoteStatusService < BaseService
-  include AuthorExtractor
-
   def call(url, prefetched_body = nil, protocol = :ostatus)
     if prefetched_body.nil?
       resource_url, resource_options, protocol = FetchAtomService.new.call(url)
@@ -12,34 +10,8 @@ class FetchRemoteStatusService < BaseService
     end
 
     case protocol
-    when :ostatus
-      process_atom(resource_url, **resource_options)
     when :activitypub
       ActivityPub::FetchRemoteStatusService.new.call(resource_url, **resource_options)
     end
   end
-
-  private
-
-  def process_atom(url, prefetched_body:)
-    Rails.logger.debug "Processing Atom for remote status at #{url}"
-
-    xml = Nokogiri::XML(prefetched_body)
-    xml.encoding = 'utf-8'
-
-    account = author_from_xml(xml.at_xpath('/xmlns:entry', xmlns: OStatus::TagManager::XMLNS))
-    domain  = Addressable::URI.parse(url).normalized_host
-
-    return nil unless !account.nil? && confirmed_domain?(domain, account)
-
-    statuses = ProcessFeedService.new.call(prefetched_body, account)
-    statuses.first
-  rescue Nokogiri::XML::XPath::SyntaxError
-    Rails.logger.debug 'Invalid XML or missing namespace'
-    nil
-  end
-
-  def confirmed_domain?(domain, account)
-    account.domain.nil? || domain.casecmp(account.domain).zero? || domain.casecmp(Addressable::URI.parse(account.remote_url.presence || account.uri).normalized_host).zero?
-  end
 end
diff --git a/app/services/follow_service.rb b/app/services/follow_service.rb
index 0305e2d62..8e118f5d3 100644
--- a/app/services/follow_service.rb
+++ b/app/services/follow_service.rb
@@ -13,7 +13,7 @@ class FollowService < BaseService
     target_account = ResolveAccountService.new.call(target_account, skip_webfinger: true)
 
     raise ActiveRecord::RecordNotFound if target_account.nil? || target_account.id == source_account.id || target_account.suspended?
-    raise Mastodon::NotPermittedError  if target_account.blocking?(source_account) || source_account.blocking?(target_account) || target_account.moved?
+    raise Mastodon::NotPermittedError  if target_account.blocking?(source_account) || source_account.blocking?(target_account) || target_account.moved? || (!target_account.local? && target_account.ostatus?)
 
     if source_account.following?(target_account)
       # We're already following this account, but we'll call follow! again to
@@ -32,7 +32,7 @@ class FollowService < BaseService
 
     if target_account.locked? || target_account.activitypub?
       request_follow(source_account, target_account, reblogs: reblogs)
-    else
+    elsif target_account.local?
       direct_follow(source_account, target_account, reblogs: reblogs)
     end
   end
@@ -44,9 +44,6 @@ class FollowService < BaseService
 
     if target_account.local?
       LocalNotificationWorker.perform_async(target_account.id, follow_request.id, follow_request.class.name)
-    elsif target_account.ostatus?
-      NotificationWorker.perform_async(build_follow_request_xml(follow_request), source_account.id, target_account.id)
-      AfterRemoteFollowRequestWorker.perform_async(follow_request.id)
     elsif target_account.activitypub?
       ActivityPub::DeliveryWorker.perform_async(build_json(follow_request), source_account.id, target_account.inbox_url)
     end
@@ -57,27 +54,12 @@ class FollowService < BaseService
   def direct_follow(source_account, target_account, reblogs: true)
     follow = source_account.follow!(target_account, reblogs: reblogs)
 
-    if target_account.local?
-      LocalNotificationWorker.perform_async(target_account.id, follow.id, follow.class.name)
-    else
-      Pubsubhubbub::SubscribeWorker.perform_async(target_account.id) unless target_account.subscribed?
-      NotificationWorker.perform_async(build_follow_xml(follow), source_account.id, target_account.id)
-      AfterRemoteFollowWorker.perform_async(follow.id)
-    end
-
+    LocalNotificationWorker.perform_async(target_account.id, follow.id, follow.class.name)
     MergeWorker.perform_async(target_account.id, source_account.id)
 
     follow
   end
 
-  def build_follow_request_xml(follow_request)
-    OStatus::AtomSerializer.render(OStatus::AtomSerializer.new.follow_request_salmon(follow_request))
-  end
-
-  def build_follow_xml(follow)
-    OStatus::AtomSerializer.render(OStatus::AtomSerializer.new.follow_salmon(follow))
-  end
-
   def build_json(follow_request)
     Oj.dump(serialize_payload(follow_request, ActivityPub::FollowSerializer))
   end
diff --git a/app/services/post_status_service.rb b/app/services/post_status_service.rb
index 7830aee11..34ec6d504 100644
--- a/app/services/post_status_service.rb
+++ b/app/services/post_status_service.rb
@@ -88,7 +88,6 @@ class PostStatusService < BaseService
   def postprocess_status!
     LinkCrawlWorker.perform_async(@status.id) unless @status.spoiler_text?
     DistributionWorker.perform_async(@status.id)
-    Pubsubhubbub::DistributionWorker.perform_async(@status.stream_entry.id)
     ActivityPub::DistributionWorker.perform_async(@status.id)
     PollExpirationNotifyWorker.perform_at(@status.poll.expires_at, @status.poll.id) if @status.poll
   end
diff --git a/app/services/process_feed_service.rb b/app/services/process_feed_service.rb
deleted file mode 100644
index 30a9dd85e..000000000
--- a/app/services/process_feed_service.rb
+++ /dev/null
@@ -1,31 +0,0 @@
-# frozen_string_literal: true
-
-class ProcessFeedService < BaseService
-  def call(body, account, **options)
-    @options = options
-
-    xml = Nokogiri::XML(body)
-    xml.encoding = 'utf-8'
-
-    update_author(body, account)
-    process_entries(xml, account)
-  end
-
-  private
-
-  def update_author(body, account)
-    RemoteProfileUpdateWorker.perform_async(account.id, body.force_encoding('UTF-8'), true)
-  end
-
-  def process_entries(xml, account)
-    xml.xpath('//xmlns:entry', xmlns: OStatus::TagManager::XMLNS).reverse_each.map { |entry| process_entry(entry, account) }.compact
-  end
-
-  def process_entry(xml, account)
-    activity = OStatus::Activity::General.new(xml, account, @options)
-    activity.specialize&.perform if activity.status?
-  rescue ActiveRecord::RecordInvalid => e
-    Rails.logger.debug "Nothing was saved for #{activity.id} because: #{e}"
-    nil
-  end
-end
diff --git a/app/services/process_interaction_service.rb b/app/services/process_interaction_service.rb
deleted file mode 100644
index 1fca3832b..000000000
--- a/app/services/process_interaction_service.rb
+++ /dev/null
@@ -1,151 +0,0 @@
-# frozen_string_literal: true
-
-class ProcessInteractionService < BaseService
-  include AuthorExtractor
-  include Authorization
-
-  # Record locally the remote interaction with our user
-  # @param [String] envelope Salmon envelope
-  # @param [Account] target_account Account the Salmon was addressed to
-  def call(envelope, target_account)
-    body = salmon.unpack(envelope)
-
-    xml = Nokogiri::XML(body)
-    xml.encoding = 'utf-8'
-
-    account = author_from_xml(xml.at_xpath('/xmlns:entry', xmlns: OStatus::TagManager::XMLNS))
-
-    return if account.nil? || account.suspended?
-
-    if salmon.verify(envelope, account.keypair)
-      RemoteProfileUpdateWorker.perform_async(account.id, body.force_encoding('UTF-8'), true)
-
-      case verb(xml)
-      when :follow
-        follow!(account, target_account) unless target_account.locked? || target_account.blocking?(account) || target_account.domain_blocking?(account.domain)
-      when :request_friend
-        follow_request!(account, target_account) unless !target_account.locked? || target_account.blocking?(account) || target_account.domain_blocking?(account.domain)
-      when :authorize
-        authorize_follow_request!(account, target_account)
-      when :reject
-        reject_follow_request!(account, target_account)
-      when :unfollow
-        unfollow!(account, target_account)
-      when :favorite
-        favourite!(xml, account)
-      when :unfavorite
-        unfavourite!(xml, account)
-      when :post
-        add_post!(body, account) if mentions_account?(xml, target_account)
-      when :share
-        add_post!(body, account) unless status(xml).nil?
-      when :delete
-        delete_post!(xml, account)
-      when :block
-        reflect_block!(account, target_account)
-      when :unblock
-        reflect_unblock!(account, target_account)
-      end
-    end
-  rescue HTTP::Error, OStatus2::BadSalmonError, Mastodon::NotPermittedError
-    nil
-  end
-
-  private
-
-  def mentions_account?(xml, account)
-    xml.xpath('/xmlns:entry/xmlns:link[@rel="mentioned"]', xmlns: OStatus::TagManager::XMLNS).each { |mention_link| return true if [OStatus::TagManager.instance.uri_for(account), OStatus::TagManager.instance.url_for(account)].include?(mention_link.attribute('href').value) }
-    false
-  end
-
-  def verb(xml)
-    raw = xml.at_xpath('//activity:verb', activity: OStatus::TagManager::AS_XMLNS).content
-    OStatus::TagManager::VERBS.key(raw)
-  rescue
-    :post
-  end
-
-  def follow!(account, target_account)
-    follow = account.follow!(target_account)
-    FollowRequest.find_by(account: account, target_account: target_account)&.destroy
-    NotifyService.new.call(target_account, follow)
-  end
-
-  def follow_request!(account, target_account)
-    return if account.requested?(target_account)
-
-    follow_request = FollowRequest.create!(account: account, target_account: target_account)
-    NotifyService.new.call(target_account, follow_request)
-  end
-
-  def authorize_follow_request!(account, target_account)
-    follow_request = FollowRequest.find_by(account: target_account, target_account: account)
-    follow_request&.authorize!
-    Pubsubhubbub::SubscribeWorker.perform_async(account.id) unless account.subscribed?
-  end
-
-  def reject_follow_request!(account, target_account)
-    follow_request = FollowRequest.find_by(account: target_account, target_account: account)
-    follow_request&.reject!
-  end
-
-  def unfollow!(account, target_account)
-    account.unfollow!(target_account)
-    FollowRequest.find_by(account: account, target_account: target_account)&.destroy
-  end
-
-  def reflect_block!(account, target_account)
-    UnfollowService.new.call(target_account, account) if target_account.following?(account)
-    account.block!(target_account)
-  end
-
-  def reflect_unblock!(account, target_account)
-    UnblockService.new.call(account, target_account)
-  end
-
-  def delete_post!(xml, account)
-    status = Status.find(xml.at_xpath('//xmlns:id', xmlns: OStatus::TagManager::XMLNS).content)
-
-    return if status.nil?
-
-    authorize_with account, status, :destroy?
-
-    RemovalWorker.perform_async(status.id)
-  end
-
-  def favourite!(xml, from_account)
-    current_status = status(xml)
-
-    return if current_status.nil?
-
-    favourite = current_status.favourites.where(account: from_account).first_or_create!(account: from_account)
-    NotifyService.new.call(current_status.account, favourite)
-  end
-
-  def unfavourite!(xml, from_account)
-    current_status = status(xml)
-
-    return if current_status.nil?
-
-    favourite = current_status.favourites.where(account: from_account).first
-    favourite&.destroy
-  end
-
-  def add_post!(body, account)
-    ProcessingWorker.perform_async(account.id, body.force_encoding('UTF-8'))
-  end
-
-  def status(xml)
-    uri = activity_id(xml)
-    return nil unless OStatus::TagManager.instance.local_id?(uri)
-    Status.find(OStatus::TagManager.instance.unique_tag_to_local_id(uri, 'Status'))
-  end
-
-  def activity_id(xml)
-    xml.at_xpath('//activity:object', activity: OStatus::TagManager::AS_XMLNS).at_xpath('./xmlns:id', xmlns: OStatus::TagManager::XMLNS).content
-  end
-
-  def salmon
-    @salmon ||= OStatus2::Salmon.new
-  end
-end
diff --git a/app/services/process_mentions_service.rb b/app/services/process_mentions_service.rb
index bc607dff3..da52bff6a 100644
--- a/app/services/process_mentions_service.rb
+++ b/app/services/process_mentions_service.rb
@@ -1,7 +1,6 @@
 # frozen_string_literal: true
 
 class ProcessMentionsService < BaseService
-  include StreamEntryRenderer
   include Payloadable
 
   # Scan status for mentions and fetch remote mentioned users, create
@@ -49,17 +48,11 @@ class ProcessMentionsService < BaseService
 
     if mentioned_account.local?
       LocalNotificationWorker.perform_async(mentioned_account.id, mention.id, mention.class.name)
-    elsif mentioned_account.ostatus? && !@status.stream_entry.hidden?
-      NotificationWorker.perform_async(ostatus_xml, @status.account_id, mentioned_account.id)
     elsif mentioned_account.activitypub?
       ActivityPub::DeliveryWorker.perform_async(activitypub_json, mention.status.account_id, mentioned_account.inbox_url)
     end
   end
 
-  def ostatus_xml
-    @ostatus_xml ||= stream_entry_to_xml(@status.stream_entry)
-  end
-
   def activitypub_json
     return @activitypub_json if defined?(@activitypub_json)
     @activitypub_json = Oj.dump(serialize_payload(@status, ActivityPub::ActivitySerializer, signer: @status.account))
diff --git a/app/services/pubsubhubbub/subscribe_service.rb b/app/services/pubsubhubbub/subscribe_service.rb
deleted file mode 100644
index 550da6328..000000000
--- a/app/services/pubsubhubbub/subscribe_service.rb
+++ /dev/null
@@ -1,53 +0,0 @@
-# frozen_string_literal: true
-
-class Pubsubhubbub::SubscribeService < BaseService
-  URL_PATTERN = /\A#{URI.regexp(%w(http https))}\z/
-
-  attr_reader :account, :callback, :secret,
-              :lease_seconds, :domain
-
-  def call(account, callback, secret, lease_seconds, verified_domain = nil)
-    @account       = account
-    @callback      = Addressable::URI.parse(callback).normalize.to_s
-    @secret        = secret
-    @lease_seconds = lease_seconds
-    @domain        = verified_domain
-
-    process_subscribe
-  end
-
-  private
-
-  def process_subscribe
-    if account.nil?
-      ['Invalid topic URL', 422]
-    elsif !valid_callback?
-      ['Invalid callback URL', 422]
-    elsif blocked_domain?
-      ['Callback URL not allowed', 403]
-    else
-      confirm_subscription
-      ['', 202]
-    end
-  end
-
-  def confirm_subscription
-    subscription = locate_subscription
-    Pubsubhubbub::ConfirmationWorker.perform_async(subscription.id, 'subscribe', secret, lease_seconds)
-  end
-
-  def valid_callback?
-    callback.present? && callback =~ URL_PATTERN
-  end
-
-  def blocked_domain?
-    DomainBlock.blocked? Addressable::URI.parse(callback).host
-  end
-
-  def locate_subscription
-    subscription = Subscription.find_or_initialize_by(account: account, callback_url: callback)
-    subscription.domain = domain
-    subscription.save!
-    subscription
-  end
-end
diff --git a/app/services/pubsubhubbub/unsubscribe_service.rb b/app/services/pubsubhubbub/unsubscribe_service.rb
deleted file mode 100644
index 646150f7b..000000000
--- a/app/services/pubsubhubbub/unsubscribe_service.rb
+++ /dev/null
@@ -1,31 +0,0 @@
-# frozen_string_literal: true
-
-class Pubsubhubbub::UnsubscribeService < BaseService
-  attr_reader :account, :callback
-
-  def call(account, callback)
-    @account  = account
-    @callback = Addressable::URI.parse(callback).normalize.to_s
-
-    process_unsubscribe
-  end
-
-  private
-
-  def process_unsubscribe
-    if account.nil?
-      ['Invalid topic URL', 422]
-    else
-      confirm_unsubscribe unless subscription.nil?
-      ['', 202]
-    end
-  end
-
-  def confirm_unsubscribe
-    Pubsubhubbub::ConfirmationWorker.perform_async(subscription.id, 'unsubscribe')
-  end
-
-  def subscription
-    @_subscription ||= Subscription.find_by(account: account, callback_url: callback)
-  end
-end
diff --git a/app/services/reblog_service.rb b/app/services/reblog_service.rb
index 9cf4bc128..3bb460fca 100644
--- a/app/services/reblog_service.rb
+++ b/app/services/reblog_service.rb
@@ -2,7 +2,6 @@
 
 class ReblogService < BaseService
   include Authorization
-  include StreamEntryRenderer
   include Payloadable
 
   # Reblog a status and notify its remote author
@@ -24,7 +23,6 @@ class ReblogService < BaseService
     reblog = account.statuses.create!(reblog: reblogged_status, text: '', visibility: visibility)
 
     DistributionWorker.perform_async(reblog.id)
-    Pubsubhubbub::DistributionWorker.perform_async(reblog.stream_entry.id)
     ActivityPub::DistributionWorker.perform_async(reblog.id)
 
     create_notification(reblog)
@@ -40,8 +38,6 @@ class ReblogService < BaseService
 
     if reblogged_status.account.local?
       LocalNotificationWorker.perform_async(reblogged_status.account_id, reblog.id, reblog.class.name)
-    elsif reblogged_status.account.ostatus?
-      NotificationWorker.perform_async(stream_entry_to_xml(reblog.stream_entry), reblog.account_id, reblogged_status.account_id)
     elsif reblogged_status.account.activitypub? && !reblogged_status.account.following?(reblog.account)
       ActivityPub::DeliveryWorker.perform_async(build_json(reblog), reblog.account_id, reblogged_status.account.inbox_url)
     end
diff --git a/app/services/reject_follow_service.rb b/app/services/reject_follow_service.rb
index f87d0ba91..bc0000c8c 100644
--- a/app/services/reject_follow_service.rb
+++ b/app/services/reject_follow_service.rb
@@ -6,25 +6,17 @@ class RejectFollowService < BaseService
   def call(source_account, target_account)
     follow_request = FollowRequest.find_by!(account: source_account, target_account: target_account)
     follow_request.reject!
-    create_notification(follow_request) unless source_account.local?
+    create_notification(follow_request) if !source_account.local? && source_account.activitypub?
     follow_request
   end
 
   private
 
   def create_notification(follow_request)
-    if follow_request.account.ostatus?
-      NotificationWorker.perform_async(build_xml(follow_request), follow_request.target_account_id, follow_request.account_id)
-    elsif follow_request.account.activitypub?
-      ActivityPub::DeliveryWorker.perform_async(build_json(follow_request), follow_request.target_account_id, follow_request.account.inbox_url)
-    end
+    ActivityPub::DeliveryWorker.perform_async(build_json(follow_request), follow_request.target_account_id, follow_request.account.inbox_url)
   end
 
   def build_json(follow_request)
     Oj.dump(serialize_payload(follow_request, ActivityPub::RejectFollowSerializer))
   end
-
-  def build_xml(follow_request)
-    OStatus::AtomSerializer.render(OStatus::AtomSerializer.new.reject_follow_request_salmon(follow_request))
-  end
 end
diff --git a/app/services/remove_status_service.rb b/app/services/remove_status_service.rb
index 81adc5aae..a8c9100b3 100644
--- a/app/services/remove_status_service.rb
+++ b/app/services/remove_status_service.rb
@@ -1,7 +1,6 @@
 # frozen_string_literal: true
 
 class RemoveStatusService < BaseService
-  include StreamEntryRenderer
   include Redisable
   include Payloadable
 
@@ -78,11 +77,6 @@ class RemoveStatusService < BaseService
     target_accounts << @status.reblog.account if @status.reblog? && !@status.reblog.account.local?
     target_accounts.uniq!(&:id)
 
-    # Ostatus
-    NotificationWorker.push_bulk(target_accounts.select(&:ostatus?).uniq(&:domain)) do |target_account|
-      [salmon_xml, @account.id, target_account.id]
-    end
-
     # ActivityPub
     ActivityPub::DeliveryWorker.push_bulk(target_accounts.select(&:activitypub?).uniq(&:preferred_inbox_url)) do |target_account|
       [signed_activity_json, @account.id, target_account.preferred_inbox_url]
@@ -90,9 +84,6 @@ class RemoveStatusService < BaseService
   end
 
   def remove_from_remote_followers
-    # OStatus
-    Pubsubhubbub::RawDistributionWorker.perform_async(salmon_xml, @account.id)
-
     # ActivityPub
     ActivityPub::DeliveryWorker.push_bulk(@account.followers.inboxes) do |inbox_url|
       [signed_activity_json, @account.id, inbox_url]
@@ -111,10 +102,6 @@ class RemoveStatusService < BaseService
     end
   end
 
-  def salmon_xml
-    @salmon_xml ||= stream_entry_to_xml(@stream_entry)
-  end
-
   def signed_activity_json
     @signed_activity_json ||= Oj.dump(serialize_payload(@status, @status.reblog? ? ActivityPub::UndoAnnounceSerializer : ActivityPub::DeleteSerializer, signer: @account))
   end
diff --git a/app/services/resolve_account_service.rb b/app/services/resolve_account_service.rb
index e557706da..0ea31a0d8 100644
--- a/app/services/resolve_account_service.rb
+++ b/app/services/resolve_account_service.rb
@@ -1,11 +1,10 @@
 # frozen_string_literal: true
 
+require_relative '../models/account'
+
 class ResolveAccountService < BaseService
-  include OStatus2::MagicKey
   include JsonLdHelper
 
-  DFRN_NS = 'http://purl.org/macgirvin/dfrn/1.0'
-
   # Find or create a local account for a remote user.
   # When creating, look up the user's webfinger and fetch all
   # important information from their feed
@@ -48,18 +47,16 @@ class ResolveAccountService < BaseService
       return
     end
 
-    return if links_missing? || auto_suspend?
     return Account.find_local(@username) if TagManager.instance.local_domain?(@domain)
+    return unless activitypub_ready?
 
     RedisLock.acquire(lock_options) do |lock|
       if lock.acquired?
         @account = Account.find_remote(@username, @domain)
 
-        if activitypub_ready? || @account&.activitypub?
-          handle_activitypub
-        else
-          handle_ostatus
-        end
+        next unless @account.nil? || @account.activitypub?
+
+        handle_activitypub
       else
         raise Mastodon::RaceConditionError
       end
@@ -73,38 +70,12 @@ class ResolveAccountService < BaseService
 
   private
 
-  def links_missing?
-    !(activitypub_ready? || ostatus_ready?)
-  end
-
-  def ostatus_ready?
-    !(@webfinger.link('http://schemas.google.com/g/2010#updates-from').nil? ||
-      @webfinger.link('salmon').nil? ||
-      @webfinger.link('http://webfinger.net/rel/profile-page').nil? ||
-      @webfinger.link('magic-public-key').nil? ||
-      canonical_uri.nil? ||
-      hub_url.nil?)
-  end
-
   def webfinger_update_due?
     @account.nil? || ((!@options[:skip_webfinger] || @account.ostatus?) && @account.possibly_stale?)
   end
 
   def activitypub_ready?
-    !@webfinger.link('self').nil? &&
-      ['application/activity+json', 'application/ld+json; profile="https://www.w3.org/ns/activitystreams"'].include?(@webfinger.link('self').type) &&
-      !actor_json.nil? &&
-      actor_json['inbox'].present?
-  end
-
-  def handle_ostatus
-    create_account if @account.nil?
-    update_account
-    update_account_profile if update_profile?
-  end
-
-  def update_profile?
-    @options[:update_profile]
+    !@webfinger.link('self').nil? && ['application/activity+json', 'application/ld+json; profile="https://www.w3.org/ns/activitystreams"'].include?(@webfinger.link('self').type)
   end
 
   def handle_activitypub
@@ -115,89 +86,10 @@ class ResolveAccountService < BaseService
     nil
   end
 
-  def create_account
-    Rails.logger.debug "Creating new remote account for #{@username}@#{@domain}"
-
-    @account = Account.new(username: @username, domain: @domain)
-    @account.suspended_at = domain_block.created_at if auto_suspend?
-    @account.silenced_at  = domain_block.created_at if auto_silence?
-    @account.private_key  = nil
-  end
-
-  def update_account
-    @account.last_webfingered_at = Time.now.utc
-    @account.protocol            = :ostatus
-    @account.remote_url          = atom_url
-    @account.salmon_url          = salmon_url
-    @account.url                 = url
-    @account.public_key          = public_key
-    @account.uri                 = canonical_uri
-    @account.hub_url             = hub_url
-    @account.save!
-  end
-
-  def auto_suspend?
-    domain_block&.suspend?
-  end
-
-  def auto_silence?
-    domain_block&.silence?
-  end
-
-  def domain_block
-    return @domain_block if defined?(@domain_block)
-    @domain_block = DomainBlock.rule_for(@domain)
-  end
-
-  def atom_url
-    @atom_url ||= @webfinger.link('http://schemas.google.com/g/2010#updates-from').href
-  end
-
-  def salmon_url
-    @salmon_url ||= @webfinger.link('salmon').href
-  end
-
   def actor_url
     @actor_url ||= @webfinger.link('self').href
   end
 
-  def url
-    @url ||= @webfinger.link('http://webfinger.net/rel/profile-page').href
-  end
-
-  def public_key
-    @public_key ||= magic_key_to_pem(@webfinger.link('magic-public-key').href)
-  end
-
-  def canonical_uri
-    return @canonical_uri if defined?(@canonical_uri)
-
-    author_uri = atom.at_xpath('/xmlns:feed/xmlns:author/xmlns:uri')
-
-    if author_uri.nil?
-      owner      = atom.at_xpath('/xmlns:feed').at_xpath('./dfrn:owner', dfrn: DFRN_NS)
-      author_uri = owner.at_xpath('./xmlns:uri') unless owner.nil?
-    end
-
-    @canonical_uri = author_uri.nil? ? nil : author_uri.content
-  end
-
-  def hub_url
-    return @hub_url if defined?(@hub_url)
-
-    hubs     = atom.xpath('//xmlns:link[@rel="hub"]')
-    @hub_url = hubs.empty? || hubs.first['href'].nil? ? nil : hubs.first['href']
-  end
-
-  def atom_body
-    return @atom_body if defined?(@atom_body)
-
-    @atom_body = Request.new(:get, atom_url).perform do |response|
-      raise Mastodon::UnexpectedResponseError, response unless response.code == 200
-      response.body_with_limit
-    end
-  end
-
   def actor_json
     return @actor_json if defined?(@actor_json)
 
@@ -205,15 +97,6 @@ class ResolveAccountService < BaseService
     @actor_json = supported_context?(json) && equals_or_includes_any?(json['type'], ActivityPub::FetchRemoteAccountService::SUPPORTED_TYPES) ? json : nil
   end
 
-  def atom
-    return @atom if defined?(@atom)
-    @atom = Nokogiri::XML(atom_body)
-  end
-
-  def update_account_profile
-    RemoteProfileUpdateWorker.perform_async(@account.id, atom_body.force_encoding('UTF-8'), false)
-  end
-
   def lock_options
     { redis: Redis.current, key: "resolve:#{@username}@#{@domain}" }
   end
diff --git a/app/services/send_interaction_service.rb b/app/services/send_interaction_service.rb
deleted file mode 100644
index 3419043e5..000000000
--- a/app/services/send_interaction_service.rb
+++ /dev/null
@@ -1,39 +0,0 @@
-# frozen_string_literal: true
-
-class SendInteractionService < BaseService
-  # Send an Atom representation of an interaction to a remote Salmon endpoint
-  # @param [String] Entry XML
-  # @param [Account] source_account
-  # @param [Account] target_account
-  def call(xml, source_account, target_account)
-    @xml            = xml
-    @source_account = source_account
-    @target_account = target_account
-
-    return if !target_account.ostatus? || block_notification?
-
-    build_request.perform do |delivery|
-      raise Mastodon::UnexpectedResponseError, delivery unless delivery.code > 199 && delivery.code < 300
-    end
-  end
-
-  private
-
-  def build_request
-    request = Request.new(:post, @target_account.salmon_url, body: envelope)
-    request.add_headers('Content-Type' => 'application/magic-envelope+xml')
-    request
-  end
-
-  def envelope
-    salmon.pack(@xml, @source_account.keypair)
-  end
-
-  def block_notification?
-    DomainBlock.blocked?(@target_account.domain)
-  end
-
-  def salmon
-    @salmon ||= OStatus2::Salmon.new
-  end
-end
diff --git a/app/services/subscribe_service.rb b/app/services/subscribe_service.rb
deleted file mode 100644
index 83fd64396..000000000
--- a/app/services/subscribe_service.rb
+++ /dev/null
@@ -1,58 +0,0 @@
-# frozen_string_literal: true
-
-class SubscribeService < BaseService
-  def call(account)
-    return if account.hub_url.blank?
-
-    @account        = account
-    @account.secret = SecureRandom.hex
-
-    build_request.perform do |response|
-      if response_failed_permanently? response
-        # We're not allowed to subscribe. Fail and move on.
-        @account.secret = ''
-        @account.save!
-      elsif response_successful? response
-        # The subscription will be confirmed asynchronously.
-        @account.save!
-      else
-        # The response was either a 429 rate limit, or a 5xx error.
-        # We need to retry at a later time. Fail loudly!
-        raise Mastodon::UnexpectedResponseError, response
-      end
-    end
-  end
-
-  private
-
-  def build_request
-    request = Request.new(:post, @account.hub_url, form: subscription_params)
-    request.on_behalf_of(some_local_account) if some_local_account
-    request
-  end
-
-  def subscription_params
-    {
-      'hub.topic': @account.remote_url,
-      'hub.mode': 'subscribe',
-      'hub.callback': api_subscription_url(@account.id),
-      'hub.verify': 'async',
-      'hub.secret': @account.secret,
-      'hub.lease_seconds': 7.days.seconds,
-    }
-  end
-
-  def some_local_account
-    @some_local_account ||= Account.local.without_suspended.first
-  end
-
-  # Any response in the 3xx or 4xx range, except for 429 (rate limit)
-  def response_failed_permanently?(response)
-    (response.status.redirect? || response.status.client_error?) && !response.status.too_many_requests?
-  end
-
-  # Any response in the 2xx range
-  def response_successful?(response)
-    response.status.success?
-  end
-end
diff --git a/app/services/unblock_service.rb b/app/services/unblock_service.rb
index 95a858e9f..c263ac8af 100644
--- a/app/services/unblock_service.rb
+++ b/app/services/unblock_service.rb
@@ -7,25 +7,17 @@ class UnblockService < BaseService
     return unless account.blocking?(target_account)
 
     unblock = account.unblock!(target_account)
-    create_notification(unblock) unless target_account.local?
+    create_notification(unblock) if !target_account.local? && target_account.activitypub?
     unblock
   end
 
   private
 
   def create_notification(unblock)
-    if unblock.target_account.ostatus?
-      NotificationWorker.perform_async(build_xml(unblock), unblock.account_id, unblock.target_account_id)
-    elsif unblock.target_account.activitypub?
-      ActivityPub::DeliveryWorker.perform_async(build_json(unblock), unblock.account_id, unblock.target_account.inbox_url)
-    end
+    ActivityPub::DeliveryWorker.perform_async(build_json(unblock), unblock.account_id, unblock.target_account.inbox_url)
   end
 
   def build_json(unblock)
     Oj.dump(serialize_payload(unblock, ActivityPub::UndoBlockSerializer))
   end
-
-  def build_xml(block)
-    OStatus::AtomSerializer.render(OStatus::AtomSerializer.new.unblock_salmon(block))
-  end
 end
diff --git a/app/services/unfavourite_service.rb b/app/services/unfavourite_service.rb
index dcc890b7d..37917a64f 100644
--- a/app/services/unfavourite_service.rb
+++ b/app/services/unfavourite_service.rb
@@ -6,7 +6,7 @@ class UnfavouriteService < BaseService
   def call(account, status)
     favourite = Favourite.find_by!(account: account, status: status)
     favourite.destroy!
-    create_notification(favourite) unless status.local?
+    create_notification(favourite) if !status.account.local? && status.account.activitypub?
     favourite
   end
 
@@ -14,19 +14,10 @@ class UnfavouriteService < BaseService
 
   def create_notification(favourite)
     status = favourite.status
-
-    if status.account.ostatus?
-      NotificationWorker.perform_async(build_xml(favourite), favourite.account_id, status.account_id)
-    elsif status.account.activitypub?
-      ActivityPub::DeliveryWorker.perform_async(build_json(favourite), favourite.account_id, status.account.inbox_url)
-    end
+    ActivityPub::DeliveryWorker.perform_async(build_json(favourite), favourite.account_id, status.account.inbox_url)
   end
 
   def build_json(favourite)
     Oj.dump(serialize_payload(favourite, ActivityPub::UndoLikeSerializer))
   end
-
-  def build_xml(favourite)
-    OStatus::AtomSerializer.render(OStatus::AtomSerializer.new.unfavourite_salmon(favourite))
-  end
 end
diff --git a/app/services/unfollow_service.rb b/app/services/unfollow_service.rb
index 17dc29735..b7033d7eb 100644
--- a/app/services/unfollow_service.rb
+++ b/app/services/unfollow_service.rb
@@ -21,8 +21,8 @@ class UnfollowService < BaseService
     return unless follow
 
     follow.destroy!
-    create_notification(follow) unless @target_account.local?
-    create_reject_notification(follow) if @target_account.local? && !@source_account.local?
+    create_notification(follow) if !@target_account.local? && @target_account.activitypub?
+    create_reject_notification(follow) if @target_account.local? && !@source_account.local? && @source_account.activitypub?
     UnmergeWorker.perform_async(@target_account.id, @source_account.id)
     follow
   end
@@ -38,16 +38,10 @@ class UnfollowService < BaseService
   end
 
   def create_notification(follow)
-    if follow.target_account.ostatus?
-      NotificationWorker.perform_async(build_xml(follow), follow.account_id, follow.target_account_id)
-    elsif follow.target_account.activitypub?
-      ActivityPub::DeliveryWorker.perform_async(build_json(follow), follow.account_id, follow.target_account.inbox_url)
-    end
+    ActivityPub::DeliveryWorker.perform_async(build_json(follow), follow.account_id, follow.target_account.inbox_url)
   end
 
   def create_reject_notification(follow)
-    # Rejecting an already-existing follow request
-    return unless follow.account.activitypub?
     ActivityPub::DeliveryWorker.perform_async(build_reject_json(follow), follow.target_account_id, follow.account.inbox_url)
   end
 
@@ -58,8 +52,4 @@ class UnfollowService < BaseService
   def build_reject_json(follow)
     Oj.dump(serialize_payload(follow, ActivityPub::RejectFollowSerializer))
   end
-
-  def build_xml(follow)
-    OStatus::AtomSerializer.render(OStatus::AtomSerializer.new.unfollow_salmon(follow))
-  end
 end
diff --git a/app/services/unsubscribe_service.rb b/app/services/unsubscribe_service.rb
deleted file mode 100644
index 95c1fb4fc..000000000
--- a/app/services/unsubscribe_service.rb
+++ /dev/null
@@ -1,36 +0,0 @@
-# frozen_string_literal: true
-
-class UnsubscribeService < BaseService
-  def call(account)
-    return if account.hub_url.blank?
-
-    @account = account
-
-    begin
-      build_request.perform do |response|
-        Rails.logger.debug "PuSH unsubscribe for #{@account.acct} failed: #{response.status}" unless response.status.success?
-      end
-    rescue HTTP::Error, OpenSSL::SSL::SSLError => e
-      Rails.logger.debug "PuSH unsubscribe for #{@account.acct} failed: #{e}"
-    end
-
-    @account.secret = ''
-    @account.subscription_expires_at = nil
-    @account.save!
-  end
-
-  private
-
-  def build_request
-    Request.new(:post, @account.hub_url, form: subscription_params)
-  end
-
-  def subscription_params
-    {
-      'hub.topic': @account.remote_url,
-      'hub.mode': 'unsubscribe',
-      'hub.callback': api_subscription_url(@account.id),
-      'hub.verify': 'async',
-    }
-  end
-end
diff --git a/app/services/update_remote_profile_service.rb b/app/services/update_remote_profile_service.rb
deleted file mode 100644
index 403395a0d..000000000
--- a/app/services/update_remote_profile_service.rb
+++ /dev/null
@@ -1,66 +0,0 @@
-# frozen_string_literal: true
-
-class UpdateRemoteProfileService < BaseService
-  attr_reader :account, :remote_profile
-
-  def call(body, account, resubscribe = false)
-    @account        = account
-    @remote_profile = RemoteProfile.new(body)
-
-    return if remote_profile.root.nil?
-
-    update_account unless remote_profile.author.nil?
-
-    old_hub_url     = account.hub_url
-    account.hub_url = remote_profile.hub_link if remote_profile.hub_link.present? && remote_profile.hub_link != old_hub_url
-
-    account.save_with_optional_media!
-
-    Pubsubhubbub::SubscribeWorker.perform_async(account.id) if resubscribe && account.hub_url != old_hub_url
-  end
-
-  private
-
-  def update_account
-    account.display_name = remote_profile.display_name || ''
-    account.note         = remote_profile.note         || ''
-    account.locked       = remote_profile.locked?
-
-    if !account.suspended? && !DomainBlock.reject_media?(account.domain)
-      if remote_profile.avatar.present?
-        account.avatar_remote_url = remote_profile.avatar
-      else
-        account.avatar_remote_url = ''
-        account.avatar.destroy
-      end
-
-      if remote_profile.header.present?
-        account.header_remote_url = remote_profile.header
-      else
-        account.header_remote_url = ''
-        account.header.destroy
-      end
-
-      save_emojis if remote_profile.emojis.present?
-    end
-  end
-
-  def save_emojis
-    do_not_download = DomainBlock.reject_media?(account.domain)
-
-    return if do_not_download
-
-    remote_profile.emojis.each do |link|
-      next unless link['href'] && link['name']
-
-      shortcode = link['name'].delete(':')
-      emoji     = CustomEmoji.find_by(shortcode: shortcode, domain: account.domain)
-
-      next unless emoji.nil?
-
-      emoji = CustomEmoji.new(shortcode: shortcode, domain: account.domain)
-      emoji.image_remote_url = link['href']
-      emoji.save
-    end
-  end
-end
diff --git a/app/services/verify_salmon_service.rb b/app/services/verify_salmon_service.rb
deleted file mode 100644
index 205b35d8b..000000000
--- a/app/services/verify_salmon_service.rb
+++ /dev/null
@@ -1,26 +0,0 @@
-# frozen_string_literal: true
-
-class VerifySalmonService < BaseService
-  include AuthorExtractor
-
-  def call(payload)
-    body = salmon.unpack(payload)
-
-    xml = Nokogiri::XML(body)
-    xml.encoding = 'utf-8'
-
-    account = author_from_xml(xml.at_xpath('/xmlns:entry', xmlns: OStatus::TagManager::XMLNS))
-
-    if account.nil?
-      false
-    else
-      salmon.verify(payload, account.keypair)
-    end
-  end
-
-  private
-
-  def salmon
-    @salmon ||= OStatus2::Salmon.new
-  end
-end
diff --git a/app/views/accounts/show.html.haml b/app/views/accounts/show.html.haml
index 950e61847..de7d2a8ba 100644
--- a/app/views/accounts/show.html.haml
+++ b/app/views/accounts/show.html.haml
@@ -7,7 +7,6 @@
   - if @account.user&.setting_noindex
     %meta{ name: 'robots', content: 'noindex' }/
 
-  %link{ rel: 'salmon', href: api_salmon_url(@account.id) }/
   %link{ rel: 'alternate', type: 'application/atom+xml', href: account_url(@account, format: 'atom') }/
   %link{ rel: 'alternate', type: 'application/rss+xml', href: account_url(@account, format: 'rss') }/
   %link{ rel: 'alternate', type: 'application/activity+json', href: ActivityPub::TagManager.instance.uri_for(@account) }/
diff --git a/app/views/admin/subscriptions/_subscription.html.haml b/app/views/admin/subscriptions/_subscription.html.haml
deleted file mode 100644
index 1dec8e396..000000000
--- a/app/views/admin/subscriptions/_subscription.html.haml
+++ /dev/null
@@ -1,18 +0,0 @@
-%tr
-  %td
-    %samp= subscription.account.acct
-  %td
-    %samp= subscription.callback_url
-  %td
-    - if subscription.confirmed?
-      %i.fa.fa-check
-  %td{ style: "color: #{subscription.expired? ? 'red' : 'inherit'};" }
-    %time.time-ago{ datetime: subscription.expires_at.iso8601, title: l(subscription.expires_at) }
-      = precede subscription.expired? ? '-' : '' do
-        = time_ago_in_words(subscription.expires_at)
-  %td
-    - if subscription.last_successful_delivery_at?
-      %time.formatted{ datetime: subscription.last_successful_delivery_at.iso8601, title: l(subscription.last_successful_delivery_at) }
-        = l subscription.last_successful_delivery_at
-    - else
-      %i.fa.fa-times
diff --git a/app/views/admin/subscriptions/index.html.haml b/app/views/admin/subscriptions/index.html.haml
deleted file mode 100644
index 83704c8ee..000000000
--- a/app/views/admin/subscriptions/index.html.haml
+++ /dev/null
@@ -1,16 +0,0 @@
-- content_for :page_title do
-  = t('admin.subscriptions.title')
-
-.table-wrapper
-  %table.table
-    %thead
-      %tr
-        %th= t('admin.subscriptions.topic')
-        %th= t('admin.subscriptions.callback_url')
-        %th= t('admin.subscriptions.confirmed')
-        %th= t('admin.subscriptions.expires_in')
-        %th= t('admin.subscriptions.last_delivery')
-    %tbody
-      = render @subscriptions
-
-= paginate @subscriptions
diff --git a/app/views/well_known/webfinger/show.xml.ruby b/app/views/well_known/webfinger/show.xml.ruby
index 968c8c138..c82cdb7b3 100644
--- a/app/views/well_known/webfinger/show.xml.ruby
+++ b/app/views/well_known/webfinger/show.xml.ruby
@@ -25,11 +25,6 @@ doc << Ox::Element.new('XRD').tap do |xrd|
     link['href']     = account_url(@account)
   end
 
-  xrd << Ox::Element.new('Link').tap do |link|
-    link['rel']      = 'salmon'
-    link['href']     = api_salmon_url(@account.id)
-  end
-
   xrd << Ox::Element.new('Link').tap do |link|
     link['rel']      = 'magic-public-key'
     link['href']     = "data:application/magic-public-key,#{@account.magic_key}"
diff --git a/app/workers/after_remote_follow_request_worker.rb b/app/workers/after_remote_follow_request_worker.rb
index 84eb6ade2..ce9c65834 100644
--- a/app/workers/after_remote_follow_request_worker.rb
+++ b/app/workers/after_remote_follow_request_worker.rb
@@ -5,27 +5,5 @@ class AfterRemoteFollowRequestWorker
 
   sidekiq_options queue: 'pull', retry: 5
 
-  attr_reader :follow_request
-
-  def perform(follow_request_id)
-    @follow_request = FollowRequest.find(follow_request_id)
-    process_follow_service if processing_required?
-  rescue ActiveRecord::RecordNotFound
-    true
-  end
-
-  private
-
-  def process_follow_service
-    follow_request.destroy
-    FollowService.new.call(follow_request.account, updated_account.acct)
-  end
-
-  def processing_required?
-    !updated_account.nil? && !updated_account.locked?
-  end
-
-  def updated_account
-    @_updated_account ||= FetchRemoteAccountService.new.call(follow_request.target_account.remote_url)
-  end
+  def perform(follow_request_id); end
 end
diff --git a/app/workers/after_remote_follow_worker.rb b/app/workers/after_remote_follow_worker.rb
index edab83f85..d9719f2bf 100644
--- a/app/workers/after_remote_follow_worker.rb
+++ b/app/workers/after_remote_follow_worker.rb
@@ -5,27 +5,5 @@ class AfterRemoteFollowWorker
 
   sidekiq_options queue: 'pull', retry: 5
 
-  attr_reader :follow
-
-  def perform(follow_id)
-    @follow = Follow.find(follow_id)
-    process_follow_service if processing_required?
-  rescue ActiveRecord::RecordNotFound
-    true
-  end
-
-  private
-
-  def process_follow_service
-    follow.destroy
-    FollowService.new.call(follow.account, updated_account.acct)
-  end
-
-  def updated_account
-    @_updated_account ||= FetchRemoteAccountService.new.call(follow.target_account.remote_url)
-  end
-
-  def processing_required?
-    !updated_account.nil? && updated_account.locked?
-  end
+  def perform(follow_id); end
 end
diff --git a/app/workers/notification_worker.rb b/app/workers/notification_worker.rb
index da1d6ab45..1c0f001cf 100644
--- a/app/workers/notification_worker.rb
+++ b/app/workers/notification_worker.rb
@@ -5,7 +5,5 @@ class NotificationWorker
 
   sidekiq_options queue: 'push', retry: 5
 
-  def perform(xml, source_account_id, target_account_id)
-    SendInteractionService.new.call(xml, Account.find(source_account_id), Account.find(target_account_id))
-  end
+  def perform(xml, source_account_id, target_account_id); end
 end
diff --git a/app/workers/processing_worker.rb b/app/workers/processing_worker.rb
index 978c3aba2..cf3bd8397 100644
--- a/app/workers/processing_worker.rb
+++ b/app/workers/processing_worker.rb
@@ -5,7 +5,5 @@ class ProcessingWorker
 
   sidekiq_options backtrace: true
 
-  def perform(account_id, body)
-    ProcessFeedService.new.call(body, Account.find(account_id), override_timestamps: true)
-  end
+  def perform(account_id, body); end
 end
diff --git a/app/workers/pubsubhubbub/confirmation_worker.rb b/app/workers/pubsubhubbub/confirmation_worker.rb
index c0e7b677e..783a8c95f 100644
--- a/app/workers/pubsubhubbub/confirmation_worker.rb
+++ b/app/workers/pubsubhubbub/confirmation_worker.rb
@@ -2,81 +2,8 @@
 
 class Pubsubhubbub::ConfirmationWorker
   include Sidekiq::Worker
-  include RoutingHelper
 
   sidekiq_options queue: 'push', retry: false
 
-  attr_reader :subscription, :mode, :secret, :lease_seconds
-
-  def perform(subscription_id, mode, secret = nil, lease_seconds = nil)
-    @subscription = Subscription.find(subscription_id)
-    @mode = mode
-    @secret = secret
-    @lease_seconds = lease_seconds
-    process_confirmation
-  end
-
-  private
-
-  def process_confirmation
-    prepare_subscription
-
-    callback_get_with_params
-    logger.debug "Confirming PuSH subscription for #{subscription.callback_url} with challenge #{challenge}: #{@callback_response_body}"
-
-    update_subscription
-  end
-
-  def update_subscription
-    if successful_subscribe?
-      subscription.save!
-    elsif successful_unsubscribe?
-      subscription.destroy!
-    end
-  end
-
-  def successful_subscribe?
-    subscribing? && response_matches_challenge?
-  end
-
-  def successful_unsubscribe?
-    (unsubscribing? && response_matches_challenge?) || !subscription.confirmed?
-  end
-
-  def response_matches_challenge?
-    @callback_response_body == challenge
-  end
-
-  def subscribing?
-    mode == 'subscribe'
-  end
-
-  def unsubscribing?
-    mode == 'unsubscribe'
-  end
-
-  def callback_get_with_params
-    Request.new(:get, subscription.callback_url, params: callback_params).perform do |response|
-      @callback_response_body = response.body_with_limit
-    end
-  end
-
-  def callback_params
-    {
-      'hub.topic': account_url(subscription.account, format: :atom),
-      'hub.mode': mode,
-      'hub.challenge': challenge,
-      'hub.lease_seconds': subscription.lease_seconds,
-    }
-  end
-
-  def prepare_subscription
-    subscription.secret = secret
-    subscription.lease_seconds = lease_seconds
-    subscription.confirmed = true
-  end
-
-  def challenge
-    @_challenge ||= SecureRandom.hex
-  end
+  def perform(subscription_id, mode, secret = nil, lease_seconds = nil); end
 end
diff --git a/app/workers/pubsubhubbub/delivery_worker.rb b/app/workers/pubsubhubbub/delivery_worker.rb
index 619bfa48a..1260060bd 100644
--- a/app/workers/pubsubhubbub/delivery_worker.rb
+++ b/app/workers/pubsubhubbub/delivery_worker.rb
@@ -2,80 +2,8 @@
 
 class Pubsubhubbub::DeliveryWorker
   include Sidekiq::Worker
-  include RoutingHelper
 
   sidekiq_options queue: 'push', retry: 3, dead: false
 
-  sidekiq_retry_in do |count|
-    5 * (count + 1)
-  end
-
-  attr_reader :subscription, :payload
-
-  def perform(subscription_id, payload)
-    @subscription = Subscription.find(subscription_id)
-    @payload = payload
-    process_delivery unless blocked_domain?
-  rescue => e
-    raise e.class, "Delivery failed for #{subscription&.callback_url}: #{e.message}", e.backtrace[0]
-  end
-
-  private
-
-  def process_delivery
-    callback_post_payload do |payload_delivery|
-      raise Mastodon::UnexpectedResponseError, payload_delivery unless response_successful? payload_delivery
-    end
-
-    subscription.touch(:last_successful_delivery_at)
-  end
-
-  def callback_post_payload(&block)
-    request = Request.new(:post, subscription.callback_url, body: payload)
-    request.add_headers(headers)
-    request.perform(&block)
-  end
-
-  def blocked_domain?
-    DomainBlock.blocked?(host)
-  end
-
-  def host
-    Addressable::URI.parse(subscription.callback_url).normalized_host
-  end
-
-  def headers
-    {
-      'Content-Type' => 'application/atom+xml',
-      'Link' => link_header,
-    }.merge(signature_headers.to_h)
-  end
-
-  def link_header
-    LinkHeader.new([hub_link_header, self_link_header]).to_s
-  end
-
-  def hub_link_header
-    [api_push_url, [%w(rel hub)]]
-  end
-
-  def self_link_header
-    [account_url(subscription.account, format: :atom), [%w(rel self)]]
-  end
-
-  def signature_headers
-    { 'X-Hub-Signature' => payload_signature } if subscription.secret?
-  end
-
-  def payload_signature
-    "sha1=#{hmac_payload_digest}"
-  end
-
-  def hmac_payload_digest
-    OpenSSL::HMAC.hexdigest(OpenSSL::Digest.new('sha1'), subscription.secret, payload)
-  end
-
-  def response_successful?(payload_delivery)
-    payload_delivery.code > 199 && payload_delivery.code < 300
-  end
+  def perform(subscription_id, payload); end
 end
diff --git a/app/workers/pubsubhubbub/distribution_worker.rb b/app/workers/pubsubhubbub/distribution_worker.rb
index fed5e917d..75bac5d6f 100644
--- a/app/workers/pubsubhubbub/distribution_worker.rb
+++ b/app/workers/pubsubhubbub/distribution_worker.rb
@@ -5,28 +5,5 @@ class Pubsubhubbub::DistributionWorker
 
   sidekiq_options queue: 'push'
 
-  def perform(stream_entry_ids)
-    stream_entries = StreamEntry.where(id: stream_entry_ids).includes(:status).reject { |e| e.status.nil? || e.status.hidden? }
-
-    return if stream_entries.empty?
-
-    @account       = stream_entries.first.account
-    @subscriptions = active_subscriptions.to_a
-
-    distribute_public!(stream_entries)
-  end
-
-  private
-
-  def distribute_public!(stream_entries)
-    @payload = OStatus::AtomSerializer.render(OStatus::AtomSerializer.new.feed(@account, stream_entries))
-
-    Pubsubhubbub::DeliveryWorker.push_bulk(@subscriptions) do |subscription_id|
-      [subscription_id, @payload]
-    end
-  end
-
-  def active_subscriptions
-    Subscription.where(account: @account).active.pluck(:id)
-  end
+  def perform(stream_entry_ids); end
 end
diff --git a/app/workers/pubsubhubbub/raw_distribution_worker.rb b/app/workers/pubsubhubbub/raw_distribution_worker.rb
index 16962a623..ece9c80ac 100644
--- a/app/workers/pubsubhubbub/raw_distribution_worker.rb
+++ b/app/workers/pubsubhubbub/raw_distribution_worker.rb
@@ -5,18 +5,5 @@ class Pubsubhubbub::RawDistributionWorker
 
   sidekiq_options queue: 'push'
 
-  def perform(xml, source_account_id)
-    @account       = Account.find(source_account_id)
-    @subscriptions = active_subscriptions.to_a
-
-    Pubsubhubbub::DeliveryWorker.push_bulk(@subscriptions) do |subscription|
-      [subscription.id, xml]
-    end
-  end
-
-  private
-
-  def active_subscriptions
-    Subscription.where(account: @account).active.select('id, callback_url, domain')
-  end
+  def perform(xml, source_account_id); end
 end
diff --git a/app/workers/pubsubhubbub/subscribe_worker.rb b/app/workers/pubsubhubbub/subscribe_worker.rb
index 2e176d1c1..b861b5e67 100644
--- a/app/workers/pubsubhubbub/subscribe_worker.rb
+++ b/app/workers/pubsubhubbub/subscribe_worker.rb
@@ -5,30 +5,5 @@ class Pubsubhubbub::SubscribeWorker
 
   sidekiq_options queue: 'push', retry: 10, unique: :until_executed, dead: false
 
-  sidekiq_retry_in do |count|
-    case count
-    when 0
-      30.minutes.seconds
-    when 1
-      2.hours.seconds
-    when 2
-      12.hours.seconds
-    else
-      24.hours.seconds * (count - 2)
-    end
-  end
-
-  sidekiq_retries_exhausted do |msg, _e|
-    account = Account.find(msg['args'].first)
-    Sidekiq.logger.error "PuSH subscription attempts for #{account.acct} exhausted. Unsubscribing"
-    ::UnsubscribeService.new.call(account)
-  end
-
-  def perform(account_id)
-    account = Account.find(account_id)
-    logger.debug "PuSH re-subscribing to #{account.acct}"
-    ::SubscribeService.new.call(account)
-  rescue => e
-    raise e.class, "Subscribe failed for #{account&.acct}: #{e.message}", e.backtrace[0]
-  end
+  def perform(account_id); end
 end
diff --git a/app/workers/pubsubhubbub/unsubscribe_worker.rb b/app/workers/pubsubhubbub/unsubscribe_worker.rb
index a271715b7..0c1c263f6 100644
--- a/app/workers/pubsubhubbub/unsubscribe_worker.rb
+++ b/app/workers/pubsubhubbub/unsubscribe_worker.rb
@@ -5,11 +5,5 @@ class Pubsubhubbub::UnsubscribeWorker
 
   sidekiq_options queue: 'push', retry: false, unique: :until_executed, dead: false
 
-  def perform(account_id)
-    account = Account.find(account_id)
-    logger.debug "PuSH unsubscribing from #{account.acct}"
-    ::UnsubscribeService.new.call(account)
-  rescue ActiveRecord::RecordNotFound
-    true
-  end
+  def perform(account_id); end
 end
diff --git a/app/workers/remote_profile_update_worker.rb b/app/workers/remote_profile_update_worker.rb
index 03585ad2d..01e8daf8f 100644
--- a/app/workers/remote_profile_update_worker.rb
+++ b/app/workers/remote_profile_update_worker.rb
@@ -5,9 +5,5 @@ class RemoteProfileUpdateWorker
 
   sidekiq_options queue: 'pull'
 
-  def perform(account_id, body, resubscribe)
-    UpdateRemoteProfileService.new.call(body, Account.find(account_id), resubscribe)
-  rescue ActiveRecord::RecordNotFound
-    true
-  end
+  def perform(account_id, body, resubscribe); end
 end
diff --git a/app/workers/salmon_worker.rb b/app/workers/salmon_worker.rb
index d37d40432..10200b06c 100644
--- a/app/workers/salmon_worker.rb
+++ b/app/workers/salmon_worker.rb
@@ -5,9 +5,5 @@ class SalmonWorker
 
   sidekiq_options backtrace: true
 
-  def perform(account_id, body)
-    ProcessInteractionService.new.call(body, Account.find(account_id))
-  rescue Nokogiri::XML::XPath::SyntaxError, ActiveRecord::RecordNotFound
-    true
-  end
+  def perform(account_id, body); end
 end
diff --git a/app/workers/scheduler/subscriptions_scheduler.rb b/app/workers/scheduler/subscriptions_scheduler.rb
index d5873bccb..6903cadc7 100644
--- a/app/workers/scheduler/subscriptions_scheduler.rb
+++ b/app/workers/scheduler/subscriptions_scheduler.rb
@@ -5,13 +5,5 @@ class Scheduler::SubscriptionsScheduler
 
   sidekiq_options unique: :until_executed, retry: 0
 
-  def perform
-    Pubsubhubbub::SubscribeWorker.push_bulk(expiring_accounts.pluck(:id))
-  end
-
-  private
-
-  def expiring_accounts
-    Account.expiring(1.day.from_now).partitioned
-  end
+  def perform; end
 end
diff --git a/config/locales/en.yml b/config/locales/en.yml
index d4f1855aa..611f36fdd 100644
--- a/config/locales/en.yml
+++ b/config/locales/en.yml
@@ -469,13 +469,6 @@ en:
       no_status_selected: No statuses were changed as none were selected
       title: Account statuses
       with_media: With media
-    subscriptions:
-      callback_url: Callback URL
-      confirmed: Confirmed
-      expires_in: Expires in
-      last_delivery: Last delivery
-      title: WebSub
-      topic: Topic
     tags:
       accounts: Accounts
       hidden: Hidden
diff --git a/config/navigation.rb b/config/navigation.rb
index df1024189..ef845d1fc 100644
--- a/config/navigation.rb
+++ b/config/navigation.rb
@@ -48,7 +48,6 @@ SimpleNavigation::Configuration.run do |navigation|
       s.item :settings, safe_join([fa_icon('cogs fw'), t('admin.settings.title')]), edit_admin_settings_url, if: -> { current_user.admin? }, highlights_on: %r{/admin/settings}
       s.item :custom_emojis, safe_join([fa_icon('smile-o fw'), t('admin.custom_emojis.title')]), admin_custom_emojis_url, highlights_on: %r{/admin/custom_emojis}
       s.item :relays, safe_join([fa_icon('exchange fw'), t('admin.relays.title')]), admin_relays_url, if: -> { current_user.admin? }, highlights_on: %r{/admin/relays}
-      s.item :subscriptions, safe_join([fa_icon('paper-plane-o fw'), t('admin.subscriptions.title')]), admin_subscriptions_url, if: -> { current_user.admin? }
       s.item :sidekiq, safe_join([fa_icon('diamond fw'), 'Sidekiq']), sidekiq_url, link_html: { target: 'sidekiq' }, if: -> { current_user.admin? }
       s.item :pghero, safe_join([fa_icon('database fw'), 'PgHero']), pghero_url, link_html: { target: 'pghero' }, if: -> { current_user.admin? }
     end
diff --git a/config/routes.rb b/config/routes.rb
index 9ab5ba7f0..4b6d464c6 100644
--- a/config/routes.rb
+++ b/config/routes.rb
@@ -154,7 +154,6 @@ Rails.application.routes.draw do
   namespace :admin do
     get '/dashboard', to: 'dashboard#index'
 
-    resources :subscriptions, only: [:index]
     resources :domain_blocks, only: [:new, :create, :show, :destroy]
     resources :email_domain_blocks, only: [:index, :new, :create, :destroy]
     resources :action_logs, only: [:index]
@@ -191,8 +190,6 @@ Rails.application.routes.draw do
 
     resources :accounts, only: [:index, :show] do
       member do
-        post :subscribe
-        post :unsubscribe
         post :enable
         post :unsilence
         post :unsuspend
@@ -257,16 +254,6 @@ Rails.application.routes.draw do
   get '/admin', to: redirect('/admin/dashboard', status: 302)
 
   namespace :api do
-    # PubSubHubbub outgoing subscriptions
-    resources :subscriptions, only: [:show]
-    post '/subscriptions/:id', to: 'subscriptions#update'
-
-    # PubSubHubbub incoming subscriptions
-    post '/push', to: 'push#update', as: :push
-
-    # Salmon
-    post '/salmon/:id', to: 'salmon#update', as: :salmon
-
     # OEmbed
     get '/oembed', to: 'oembed#show', as: :oembed
 
@@ -318,7 +305,6 @@ Rails.application.routes.draw do
 
       get '/search', to: 'search#index', as: :search
 
-      resources :follows,      only: [:create]
       resources :media,        only: [:create, :update]
       resources :blocks,       only: [:index]
       resources :mutes,        only: [:index]
diff --git a/config/sidekiq.yml b/config/sidekiq.yml
index 0ec1742ab..a16dea967 100644
--- a/config/sidekiq.yml
+++ b/config/sidekiq.yml
@@ -9,9 +9,6 @@
   scheduled_statuses_scheduler:
     every: '5m'
     class: Scheduler::ScheduledStatusesScheduler
-  subscriptions_scheduler:
-    cron: '<%= Random.rand(0..59) %> <%= Random.rand(4..6) %> * * *'
-    class: Scheduler::SubscriptionsScheduler
   media_cleanup_scheduler:
     cron: '<%= Random.rand(0..59) %> <%= Random.rand(3..5) %> * * *'
     class: Scheduler::MediaCleanupScheduler
diff --git a/spec/controllers/admin/accounts_controller_spec.rb b/spec/controllers/admin/accounts_controller_spec.rb
index a348ab3d7..608606ff9 100644
--- a/spec/controllers/admin/accounts_controller_spec.rb
+++ b/spec/controllers/admin/accounts_controller_spec.rb
@@ -75,44 +75,6 @@ RSpec.describe Admin::AccountsController, type: :controller do
     end
   end
 
-  describe 'POST #subscribe' do
-    subject { post :subscribe, params: { id: account.id } }
-
-    let(:current_user) { Fabricate(:user, admin: admin) }
-    let(:account) { Fabricate(:account) }
-
-    context 'when user is admin' do
-      let(:admin) { true }
-
-      it { is_expected.to redirect_to admin_account_path(account.id) }
-    end
-
-    context 'when user is not admin' do
-      let(:admin) { false }
-
-      it { is_expected.to have_http_status :forbidden }
-    end
-  end
-
-  describe 'POST #unsubscribe' do
-    subject { post :unsubscribe, params: { id: account.id } }
-
-    let(:current_user) { Fabricate(:user, admin: admin) }
-    let(:account) { Fabricate(:account) }
-
-    context 'when user is admin' do
-      let(:admin) { true }
-
-      it { is_expected.to redirect_to admin_account_path(account.id) }
-    end
-
-    context 'when user is not admin' do
-      let(:admin) { false }
-
-      it { is_expected.to have_http_status :forbidden }
-    end
-  end
-
   describe 'POST #memorialize' do
     subject { post :memorialize, params: { id: account.id } }
 
diff --git a/spec/controllers/admin/subscriptions_controller_spec.rb b/spec/controllers/admin/subscriptions_controller_spec.rb
deleted file mode 100644
index 967152abe..000000000
--- a/spec/controllers/admin/subscriptions_controller_spec.rb
+++ /dev/null
@@ -1,32 +0,0 @@
-# frozen_string_literal: true
-require 'rails_helper'
-
-RSpec.describe Admin::SubscriptionsController, type: :controller do
-  render_views
-
-  describe 'GET #index' do
-    around do |example|
-      default_per_page = Subscription.default_per_page
-      Subscription.paginates_per 1
-      example.run
-      Subscription.paginates_per default_per_page
-    end
-
-    before do
-      sign_in Fabricate(:user, admin: true), scope: :user
-    end
-
-    it 'renders subscriptions' do
-      Fabricate(:subscription)
-      specified = Fabricate(:subscription)
-
-      get :index
-
-      subscriptions = assigns(:subscriptions)
-      expect(subscriptions.count).to eq 1
-      expect(subscriptions[0]).to eq specified
-
-      expect(response).to have_http_status(200)
-    end
-  end
-end
diff --git a/spec/controllers/api/push_controller_spec.rb b/spec/controllers/api/push_controller_spec.rb
deleted file mode 100644
index d769d8554..000000000
--- a/spec/controllers/api/push_controller_spec.rb
+++ /dev/null
@@ -1,59 +0,0 @@
-require 'rails_helper'
-
-RSpec.describe Api::PushController, type: :controller do
-  describe 'POST #update' do
-    context 'with hub.mode=subscribe' do
-      it 'creates a subscription' do
-        service = double(call: ['', 202])
-        allow(Pubsubhubbub::SubscribeService).to receive(:new).and_return(service)
-        account = Fabricate(:account)
-        account_topic_url = "https://#{Rails.configuration.x.local_domain}/users/#{account.username}.atom"
-        post :update, params: {
-          'hub.mode' => 'subscribe',
-          'hub.topic' => account_topic_url,
-          'hub.callback' => 'https://callback.host/api',
-          'hub.lease_seconds' => '3600',
-          'hub.secret' => 'as1234df',
-        }
-
-        expect(service).to have_received(:call).with(
-          account,
-          'https://callback.host/api',
-          'as1234df',
-          '3600',
-          nil
-        )
-        expect(response).to have_http_status(202)
-      end
-    end
-
-    context 'with hub.mode=unsubscribe' do
-      it 'unsubscribes the account' do
-        service = double(call: ['', 202])
-        allow(Pubsubhubbub::UnsubscribeService).to receive(:new).and_return(service)
-        account = Fabricate(:account)
-        account_topic_url = "https://#{Rails.configuration.x.local_domain}/users/#{account.username}.atom"
-        post :update, params: {
-          'hub.mode' => 'unsubscribe',
-          'hub.topic' => account_topic_url,
-          'hub.callback' => 'https://callback.host/api',
-        }
-
-        expect(service).to have_received(:call).with(
-          account,
-          'https://callback.host/api',
-        )
-        expect(response).to have_http_status(202)
-      end
-    end
-
-    context 'with unknown mode' do
-      it 'returns an unknown mode error' do
-        post :update, params: { 'hub.mode' => 'fake' }
-
-        expect(response).to have_http_status(422)
-        expect(response.body).to match(/Unknown mode/)
-      end
-    end
-  end
-end
diff --git a/spec/controllers/api/salmon_controller_spec.rb b/spec/controllers/api/salmon_controller_spec.rb
deleted file mode 100644
index 235a29af0..000000000
--- a/spec/controllers/api/salmon_controller_spec.rb
+++ /dev/null
@@ -1,65 +0,0 @@
-require 'rails_helper'
-
-RSpec.describe Api::SalmonController, type: :controller do
-  render_views
-
-  let(:account) { Fabricate(:user, account: Fabricate(:account, username: 'catsrgr8')).account }
-
-  before do
-    stub_request(:get, "https://quitter.no/.well-known/host-meta").to_return(request_fixture('.host-meta.txt'))
-    stub_request(:get, "https://quitter.no/.well-known/webfinger?resource=acct:gargron@quitter.no").to_return(request_fixture('webfinger.txt'))
-    stub_request(:get, "https://quitter.no/api/statuses/user_timeline/7477.atom").to_return(request_fixture('feed.txt'))
-    stub_request(:get, "https://quitter.no/avatar/7477-300-20160211190340.png").to_return(request_fixture('avatar.txt'))
-  end
-
-  describe 'POST #update' do
-    context 'with valid post data' do
-      before do
-        post :update, params: { id: account.id }, body: File.read(Rails.root.join('spec', 'fixtures', 'salmon', 'mention.xml'))
-      end
-
-      it 'contains XML in the request body' do
-        expect(request.body.read).to be_a String
-      end
-
-      it 'returns http success' do
-        expect(response).to have_http_status(202)
-      end
-
-      it 'creates remote account' do
-        expect(Account.find_by(username: 'gargron', domain: 'quitter.no')).to_not be_nil
-      end
-
-      it 'creates status' do
-        expect(Status.find_by(uri: 'tag:quitter.no,2016-03-20:noticeId=1276923:objectType=note')).to_not be_nil
-      end
-
-      it 'creates mention for target account' do
-        expect(account.mentions.count).to eq 1
-      end
-    end
-
-    context 'with empty post data' do
-      before do
-        post :update, params: { id: account.id }, body: ''
-      end
-
-      it 'returns http client error' do
-        expect(response).to have_http_status(400)
-      end
-    end
-
-    context 'with invalid post data' do
-      before do
-        service = double(call: false)
-        allow(VerifySalmonService).to receive(:new).and_return(service)
-
-        post :update, params: { id: account.id }, body: File.read(Rails.root.join('spec', 'fixtures', 'salmon', 'mention.xml'))
-      end
-
-      it 'returns http client error' do
-        expect(response).to have_http_status(401)
-      end
-    end
-  end
-end
diff --git a/spec/controllers/api/subscriptions_controller_spec.rb b/spec/controllers/api/subscriptions_controller_spec.rb
deleted file mode 100644
index 7a4252fe6..000000000
--- a/spec/controllers/api/subscriptions_controller_spec.rb
+++ /dev/null
@@ -1,68 +0,0 @@
-require 'rails_helper'
-
-RSpec.describe Api::SubscriptionsController, type: :controller do
-  render_views
-
-  let(:account) { Fabricate(:account, username: 'gargron', domain: 'quitter.no', remote_url: 'topic_url', secret: 'abc') }
-
-  describe 'GET #show' do
-    context 'with valid subscription' do
-      before do
-        get :show, params: { :id => account.id, 'hub.topic' => 'topic_url', 'hub.challenge' => '456', 'hub.lease_seconds' => "#{86400 * 30}" }
-      end
-
-      it 'returns http success' do
-        expect(response).to have_http_status(200)
-      end
-
-      it 'echoes back the challenge' do
-        expect(response.body).to match '456'
-      end
-    end
-
-    context 'with invalid subscription' do
-      before do
-        expect_any_instance_of(Account).to receive_message_chain(:subscription, :valid?).and_return(false)
-        get :show, params: { :id => account.id }
-      end
-
-      it 'returns http success' do
-        expect(response).to have_http_status(404)
-      end
-    end
-  end
-
-  describe 'POST #update' do
-    let(:feed) { File.read(Rails.root.join('spec', 'fixtures', 'push', 'feed.atom')) }
-
-    before do
-      stub_request(:post, "https://quitter.no/main/push/hub").to_return(:status => 200, :body => "", :headers => {})
-      stub_request(:get, "https://quitter.no/avatar/7477-300-20160211190340.png").to_return(request_fixture('avatar.txt'))
-      stub_request(:get, "https://quitter.no/notice/1269244").to_return(status: 404)
-      stub_request(:get, "https://quitter.no/notice/1265331").to_return(status: 404)
-      stub_request(:get, "https://community.highlandarrow.com/notice/54411").to_return(status: 404)
-      stub_request(:get, "https://community.highlandarrow.com/notice/53857").to_return(status: 404)
-      stub_request(:get, "https://community.highlandarrow.com/notice/51852").to_return(status: 404)
-      stub_request(:get, "https://social.umeahackerspace.se/notice/424348").to_return(status: 404)
-      stub_request(:get, "https://community.highlandarrow.com/notice/50467").to_return(status: 404)
-      stub_request(:get, "https://quitter.no/notice/1243309").to_return(status: 404)
-      stub_request(:get, "https://quitter.no/user/7477").to_return(status: 404)
-      stub_request(:any, "https://community.highlandarrow.com/user/1").to_return(status: 404)
-      stub_request(:any, "https://social.umeahackerspace.se/user/2").to_return(status: 404)
-      stub_request(:any, "https://gs.kawa-kun.com/user/2").to_return(status: 404)
-      stub_request(:any, "https://mastodon.social/users/Gargron").to_return(status: 404)
-
-      request.env['HTTP_X_HUB_SIGNATURE'] = "sha1=#{OpenSSL::HMAC.hexdigest('sha1', 'abc', feed)}"
-
-      post :update, params: { id: account.id }, body: feed
-    end
-
-    it 'returns http success' do
-      expect(response).to have_http_status(200)
-    end
-
-    it 'creates statuses for feed' do
-      expect(account.statuses.count).to_not eq 0
-    end
-  end
-end
diff --git a/spec/controllers/api/v1/follows_controller_spec.rb b/spec/controllers/api/v1/follows_controller_spec.rb
deleted file mode 100644
index 089e0fe5e..000000000
--- a/spec/controllers/api/v1/follows_controller_spec.rb
+++ /dev/null
@@ -1,51 +0,0 @@
-require 'rails_helper'
-
-RSpec.describe Api::V1::FollowsController, type: :controller do
-  render_views
-
-  let(:user)  { Fabricate(:user, account: Fabricate(:account, username: 'alice')) }
-  let(:token) { Fabricate(:accessible_access_token, resource_owner_id: user.id, scopes: 'write:follows') }
-
-  before do
-    allow(controller).to receive(:doorkeeper_token) { token }
-  end
-
-  describe 'POST #create' do
-    before do
-      stub_request(:get,  "https://quitter.no/.well-known/host-meta").to_return(request_fixture('.host-meta.txt'))
-      stub_request(:get,  "https://quitter.no/.well-known/webfinger?resource=acct:gargron@quitter.no").to_return(request_fixture('webfinger.txt'))
-      stub_request(:head, "https://quitter.no/api/statuses/user_timeline/7477.atom").to_return(:status => 405, :body => "", :headers => {})
-      stub_request(:get,  "https://quitter.no/api/statuses/user_timeline/7477.atom").to_return(request_fixture('feed.txt'))
-      stub_request(:get,  "https://quitter.no/avatar/7477-300-20160211190340.png").to_return(request_fixture('avatar.txt'))
-      stub_request(:post, "https://quitter.no/main/push/hub").to_return(:status => 200, :body => "", :headers => {})
-      stub_request(:post, "https://quitter.no/main/salmon/user/7477").to_return(:status => 200, :body => "", :headers => {})
-
-      post :create, params: { uri: 'gargron@quitter.no' }
-    end
-
-    it 'returns http success' do
-      expect(response).to have_http_status(200)
-    end
-
-    it 'creates account for remote user' do
-      expect(Account.find_by(username: 'gargron', domain: 'quitter.no')).to_not be_nil
-    end
-
-    it 'creates a follow relation between user and remote user' do
-      expect(user.account.following?(Account.find_by(username: 'gargron', domain: 'quitter.no'))).to be true
-    end
-
-    it 'sends a salmon slap to the remote user' do
-      expect(a_request(:post, "https://quitter.no/main/salmon/user/7477")).to have_been_made
-    end
-
-    it 'subscribes to remote hub' do
-      expect(a_request(:post, "https://quitter.no/main/push/hub")).to have_been_made
-    end
-
-    it 'returns http success if already following, too' do
-      post :create, params: { uri: 'gargron@quitter.no' }
-      expect(response).to have_http_status(200)
-    end
-  end
-end
diff --git a/spec/fixtures/requests/webfinger.txt b/spec/fixtures/requests/webfinger.txt
index edb8a2dbb..f337ecae6 100644
--- a/spec/fixtures/requests/webfinger.txt
+++ b/spec/fixtures/requests/webfinger.txt
@@ -8,4 +8,4 @@ Access-Control-Allow-Origin: *
 Vary: Accept-Encoding,Cookie
 Strict-Transport-Security: max-age=31536000; includeSubdomains;
 
-{"subject":"acct:gargron@quitter.no","aliases":["https:\/\/quitter.no\/user\/7477","https:\/\/quitter.no\/gargron","https:\/\/quitter.no\/index.php\/user\/7477","https:\/\/quitter.no\/index.php\/gargron"],"links":[{"rel":"http:\/\/webfinger.net\/rel\/profile-page","type":"text\/html","href":"https:\/\/quitter.no\/gargron"},{"rel":"http:\/\/gmpg.org\/xfn\/11","type":"text\/html","href":"https:\/\/quitter.no\/gargron"},{"rel":"describedby","type":"application\/rdf+xml","href":"https:\/\/quitter.no\/gargron\/foaf"},{"rel":"http:\/\/apinamespace.org\/atom","type":"application\/atomsvc+xml","href":"https:\/\/quitter.no\/api\/statusnet\/app\/service\/gargron.xml"},{"rel":"http:\/\/apinamespace.org\/twitter","href":"https:\/\/quitter.no\/api\/"},{"rel":"http:\/\/specs.openid.net\/auth\/2.0\/provider","href":"https:\/\/quitter.no\/gargron"},{"rel":"http:\/\/schemas.google.com\/g\/2010#updates-from","type":"application\/atom+xml","href":"https:\/\/quitter.no\/api\/statuses\/user_timeline\/7477.atom"},{"rel":"magic-public-key","href":"data:application\/magic-public-key,RSA.1ZBkHTavLvxH3FzlKv4O6WtlILKRFfNami3_Rcu8EuogtXSYiS-bB6hElZfUCSHbC4uLemOA34PEhz__CDMozax1iI_t8dzjDnh1x0iFSup7pSfW9iXk_WU3Dm74yWWW2jildY41vWgrEstuQ1dJ8vVFfSJ9T_tO4c-T9y8vDI8=.AQAB"},{"rel":"salmon","href":"https:\/\/quitter.no\/main\/salmon\/user\/7477"},{"rel":"http:\/\/salmon-protocol.org\/ns\/salmon-replies","href":"https:\/\/quitter.no\/main\/salmon\/user\/7477"},{"rel":"http:\/\/salmon-protocol.org\/ns\/salmon-mention","href":"https:\/\/quitter.no\/main\/salmon\/user\/7477"},{"rel":"http:\/\/ostatus.org\/schema\/1.0\/subscribe","template":"https:\/\/quitter.no\/main\/ostatussub?profile={uri}"}]}
\ No newline at end of file
+{"subject":"acct:gargron@quitter.no","aliases":["https:\/\/quitter.no\/user\/7477","https:\/\/quitter.no\/gargron","https:\/\/quitter.no\/index.php\/user\/7477","https:\/\/quitter.no\/index.php\/gargron"],"links":[{"rel":"http:\/\/webfinger.net\/rel\/profile-page","type":"text\/html","href":"https:\/\/quitter.no\/gargron"},{"rel":"http:\/\/gmpg.org\/xfn\/11","type":"text\/html","href":"https:\/\/quitter.no\/gargron"},{"rel":"describedby","type":"application\/rdf+xml","href":"https:\/\/quitter.no\/gargron\/foaf"},{"rel":"http:\/\/apinamespace.org\/atom","type":"application\/atomsvc+xml","href":"https:\/\/quitter.no\/api\/statusnet\/app\/service\/gargron.xml"},{"rel":"http:\/\/apinamespace.org\/twitter","href":"https:\/\/quitter.no\/api\/"},{"rel":"http:\/\/specs.openid.net\/auth\/2.0\/provider","href":"https:\/\/quitter.no\/gargron"},{"rel":"http:\/\/schemas.google.com\/g\/2010#updates-from","type":"application\/atom+xml","href":"https:\/\/quitter.no\/api\/statuses\/user_timeline\/7477.atom"},{"rel":"magic-public-key","href":"data:application\/magic-public-key,RSA.1ZBkHTavLvxH3FzlKv4O6WtlILKRFfNami3_Rcu8EuogtXSYiS-bB6hElZfUCSHbC4uLemOA34PEhz__CDMozax1iI_t8dzjDnh1x0iFSup7pSfW9iXk_WU3Dm74yWWW2jildY41vWgrEstuQ1dJ8vVFfSJ9T_tO4c-T9y8vDI8=.AQAB"},{"rel":"salmon","href":"https:\/\/quitter.no\/main\/salmon\/user\/7477"},{"rel":"http:\/\/salmon-protocol.org\/ns\/salmon-replies","href":"https:\/\/quitter.no\/main\/salmon\/user\/7477"},{"rel":"http:\/\/salmon-protocol.org\/ns\/salmon-mention","href":"https:\/\/quitter.no\/main\/salmon\/user\/7477"},{"rel":"http:\/\/ostatus.org\/schema\/1.0\/subscribe","template":"https:\/\/quitter.no\/main\/ostatussub?profile={uri}"}]}
diff --git a/spec/lib/ostatus/atom_serializer_spec.rb b/spec/lib/ostatus/atom_serializer_spec.rb
index 891871c1c..74ab7576f 100644
--- a/spec/lib/ostatus/atom_serializer_spec.rb
+++ b/spec/lib/ostatus/atom_serializer_spec.rb
@@ -406,28 +406,6 @@ RSpec.describe OStatus::AtomSerializer do
         scope = entry.nodes.find { |node| node.name == 'mastodon:scope' }
         expect(scope.text).to eq 'public'
       end
-
-      it 'returns element whose rendered view triggers creation when processed' do
-        remote_account = Account.create!(username: 'username')
-        remote_status = Fabricate(:status, account: remote_account, created_at: '2000-01-01T00:00:00Z')
-
-        entry = OStatus::AtomSerializer.new.entry(remote_status.stream_entry, true)
-        entry.nodes.delete_if { |node| node[:type] == 'application/activity+json' } # Remove ActivityPub link to simplify test
-        xml = OStatus::AtomSerializer.render(entry).gsub('cb6e6126.ngrok.io', 'remote.test')
-
-        remote_status.destroy!
-        remote_account.destroy!
-
-        account = Account.create!(
-          domain: 'remote.test',
-          username: 'username',
-          last_webfingered_at: Time.now.utc
-        )
-
-        ProcessFeedService.new.call(xml, account)
-
-        expect(Status.find_by(uri: "https://remote.test/users/#{remote_status.account.to_param}/statuses/#{remote_status.id}")).to be_instance_of Status
-      end
     end
 
     context 'if status is not present' do
@@ -683,24 +661,6 @@ RSpec.describe OStatus::AtomSerializer do
       end
     end
 
-    it 'appends link element for hub' do
-      account = Fabricate(:account, username: 'username')
-
-      feed = OStatus::AtomSerializer.new.feed(account, [])
-
-      link = feed.nodes.find { |node| node.name == 'link' && node[:rel] == 'hub' }
-      expect(link[:href]).to eq 'https://cb6e6126.ngrok.io/api/push'
-    end
-
-    it 'appends link element for Salmon' do
-      account = Fabricate(:account, username: 'username')
-
-      feed = OStatus::AtomSerializer.new.feed(account, [])
-
-      link = feed.nodes.find { |node| node.name == 'link' && node[:rel] == 'salmon' }
-      expect(link[:href]).to start_with 'https://cb6e6126.ngrok.io/api/salmon/'
-    end
-
     it 'appends stream entries' do
       account = Fabricate(:account, username: 'username')
       status = Fabricate(:status, account: account)
@@ -784,18 +744,6 @@ RSpec.describe OStatus::AtomSerializer do
       object = block_salmon.nodes.find { |node| node.name == 'activity:object' }
       expect(object.id.text).to eq 'https://domain.test/id'
     end
-
-    it 'returns element whose rendered view triggers block when processed' do
-      block = Fabricate(:block)
-      block_salmon = OStatus::AtomSerializer.new.block_salmon(block)
-      xml = OStatus::AtomSerializer.render(block_salmon)
-      envelope = OStatus2::Salmon.new.pack(xml, block.account.keypair)
-      block.destroy!
-
-      ProcessInteractionService.new.call(envelope, block.target_account)
-
-      expect(block.account.blocking?(block.target_account)).to be true
-    end
   end
 
   describe '#unblock_salmon' do
@@ -871,17 +819,6 @@ RSpec.describe OStatus::AtomSerializer do
       object = unblock_salmon.nodes.find { |node| node.name == 'activity:object' }
       expect(object.id.text).to eq 'https://domain.test/id'
     end
-
-    it 'returns element whose rendered view triggers block when processed' do
-      block = Fabricate(:block)
-      unblock_salmon = OStatus::AtomSerializer.new.unblock_salmon(block)
-      xml = OStatus::AtomSerializer.render(unblock_salmon)
-      envelope = OStatus2::Salmon.new.pack(xml, block.account.keypair)
-
-      ProcessInteractionService.new.call(envelope, block.target_account)
-
-      expect { block.reload }.to raise_error ActiveRecord::RecordNotFound
-    end
   end
 
   describe '#favourite_salmon' do
@@ -964,17 +901,6 @@ RSpec.describe OStatus::AtomSerializer do
       expect(favourite_salmon.title.text).to eq 'account favourited a status by status_account@remote'
       expect(favourite_salmon.content.text).to eq 'account favourited a status by status_account@remote'
     end
-
-    it 'returns element whose rendered view triggers favourite when processed' do
-      favourite = Fabricate(:favourite)
-      favourite_salmon = OStatus::AtomSerializer.new.favourite_salmon(favourite)
-      xml = OStatus::AtomSerializer.render(favourite_salmon)
-      envelope = OStatus2::Salmon.new.pack(xml, favourite.account.keypair)
-      favourite.destroy!
-
-      ProcessInteractionService.new.call(envelope, favourite.status.account)
-      expect(favourite.account.favourited?(favourite.status)).to be true
-    end
   end
 
   describe '#unfavourite_salmon' do
@@ -1064,16 +990,6 @@ RSpec.describe OStatus::AtomSerializer do
       expect(unfavourite_salmon.title.text).to eq 'account no longer favourites a status by status_account@remote'
       expect(unfavourite_salmon.content.text).to eq 'account no longer favourites a status by status_account@remote'
     end
-
-    it 'returns element whose rendered view triggers unfavourite when processed' do
-      favourite = Fabricate(:favourite)
-      unfavourite_salmon = OStatus::AtomSerializer.new.unfavourite_salmon(favourite)
-      xml = OStatus::AtomSerializer.render(unfavourite_salmon)
-      envelope = OStatus2::Salmon.new.pack(xml, favourite.account.keypair)
-
-      ProcessInteractionService.new.call(envelope, favourite.status.account)
-      expect { favourite.reload }.to raise_error ActiveRecord::RecordNotFound
-    end
   end
 
   describe '#follow_salmon' do
@@ -1143,18 +1059,6 @@ RSpec.describe OStatus::AtomSerializer do
       expect(follow_salmon.title.text).to eq 'account started following target_account@remote'
       expect(follow_salmon.content.text).to eq 'account started following target_account@remote'
     end
-
-    it 'returns element whose rendered view triggers follow when processed' do
-      follow = Fabricate(:follow)
-      follow_salmon = OStatus::AtomSerializer.new.follow_salmon(follow)
-      xml = OStatus::AtomSerializer.render(follow_salmon)
-      follow.destroy!
-      envelope = OStatus2::Salmon.new.pack(xml, follow.account.keypair)
-
-      ProcessInteractionService.new.call(envelope, follow.target_account)
-
-      expect(follow.account.following?(follow.target_account)).to be true
-    end
   end
 
   describe '#unfollow_salmon' do
@@ -1251,19 +1155,6 @@ RSpec.describe OStatus::AtomSerializer do
       object = unfollow_salmon.nodes.find { |node| node.name == 'activity:object' }
       expect(object.id.text).to eq 'https://domain.test/id'
     end
-
-    it 'returns element whose rendered view triggers unfollow when processed' do
-      follow = Fabricate(:follow)
-      follow.destroy!
-      unfollow_salmon = OStatus::AtomSerializer.new.unfollow_salmon(follow)
-      xml = OStatus::AtomSerializer.render(unfollow_salmon)
-      follow.account.follow!(follow.target_account)
-      envelope = OStatus2::Salmon.new.pack(xml, follow.account.keypair)
-
-      ProcessInteractionService.new.call(envelope, follow.target_account)
-
-      expect(follow.account.following?(follow.target_account)).to be false
-    end
   end
 
   describe '#follow_request_salmon' do
@@ -1294,18 +1185,6 @@ RSpec.describe OStatus::AtomSerializer do
         follow_request_salmon = serialize(follow_request)
         expect(follow_request_salmon.title.text).to eq 'account requested to follow target_account@remote'
       end
-
-      it 'returns element whose rendered view triggers follow request when processed' do
-        follow_request = Fabricate(:follow_request)
-        follow_request_salmon = serialize(follow_request)
-        xml = OStatus::AtomSerializer.render(follow_request_salmon)
-        envelope = OStatus2::Salmon.new.pack(xml, follow_request.account.keypair)
-        follow_request.destroy!
-
-        ProcessInteractionService.new.call(envelope, follow_request.target_account)
-
-        expect(follow_request.account.requested?(follow_request.target_account)).to eq true
-      end
     end
   end
 
@@ -1364,18 +1243,6 @@ RSpec.describe OStatus::AtomSerializer do
       verb = authorize_follow_request_salmon.nodes.find { |node| node.name == 'activity:verb' }
       expect(verb.text).to eq OStatus::TagManager::VERBS[:authorize]
     end
-
-    it 'returns element whose rendered view creates follow from follow request when processed' do
-      follow_request = Fabricate(:follow_request)
-      authorize_follow_request_salmon = OStatus::AtomSerializer.new.authorize_follow_request_salmon(follow_request)
-      xml = OStatus::AtomSerializer.render(authorize_follow_request_salmon)
-      envelope = OStatus2::Salmon.new.pack(xml, follow_request.target_account.keypair)
-
-      ProcessInteractionService.new.call(envelope, follow_request.account)
-
-      expect(follow_request.account.following?(follow_request.target_account)).to eq true
-      expect { follow_request.reload }.to raise_error ActiveRecord::RecordNotFound
-    end
   end
 
   describe '#reject_follow_request_salmon' do
@@ -1427,18 +1294,6 @@ RSpec.describe OStatus::AtomSerializer do
       verb = reject_follow_request_salmon.nodes.find { |node| node.name == 'activity:verb' }
       expect(verb.text).to eq OStatus::TagManager::VERBS[:reject]
     end
-
-    it 'returns element whose rendered view deletes follow request when processed' do
-      follow_request = Fabricate(:follow_request)
-      reject_follow_request_salmon = OStatus::AtomSerializer.new.reject_follow_request_salmon(follow_request)
-      xml = OStatus::AtomSerializer.render(reject_follow_request_salmon)
-      envelope = OStatus2::Salmon.new.pack(xml, follow_request.target_account.keypair)
-
-      ProcessInteractionService.new.call(envelope, follow_request.account)
-
-      expect(follow_request.account.following?(follow_request.target_account)).to eq false
-      expect { follow_request.reload }.to raise_error ActiveRecord::RecordNotFound
-    end
   end
 
   describe '#object' do
diff --git a/spec/services/authorize_follow_service_spec.rb b/spec/services/authorize_follow_service_spec.rb
index 562ef0041..ce56d57a6 100644
--- a/spec/services/authorize_follow_service_spec.rb
+++ b/spec/services/authorize_follow_service_spec.rb
@@ -38,13 +38,6 @@ RSpec.describe AuthorizeFollowService, type: :service do
     it 'creates follow relation' do
       expect(bob.following?(sender)).to be true
     end
-
-    it 'sends a follow request authorization salmon slap' do
-      expect(a_request(:post, "http://salmon.example.com/").with { |req|
-        xml = OStatus2::Salmon.new.unpack(req.body)
-        xml.match(OStatus::TagManager::VERBS[:authorize])
-      }).to have_been_made.once
-    end
   end
 
   describe 'remote ActivityPub' do
diff --git a/spec/services/batched_remove_status_service_spec.rb b/spec/services/batched_remove_status_service_spec.rb
index e53623449..d52e7f484 100644
--- a/spec/services/batched_remove_status_service_spec.rb
+++ b/spec/services/batched_remove_status_service_spec.rb
@@ -49,19 +49,6 @@ RSpec.describe BatchedRemoveStatusService, type: :service do
     expect(Redis.current).to have_received(:publish).with('timeline:public', any_args).at_least(:once)
   end
 
-  it 'sends PuSH update to PuSH subscribers' do
-    expect(a_request(:post, 'http://example.com/push').with { |req|
-      matches = req.body.match(OStatus::TagManager::VERBS[:delete])
-    }).to have_been_made.at_least_once
-  end
-
-  it 'sends Salmon slap to previously mentioned users' do
-    expect(a_request(:post, "http://example.com/salmon").with { |req|
-      xml = OStatus2::Salmon.new.unpack(req.body)
-      xml.match(OStatus::TagManager::VERBS[:delete])
-    }).to have_been_made.once
-  end
-
   it 'sends delete activity to followers' do
     expect(a_request(:post, 'http://example.com/inbox')).to have_been_made.at_least_once
   end
diff --git a/spec/services/block_service_spec.rb b/spec/services/block_service_spec.rb
index 6584bb90e..de20dd026 100644
--- a/spec/services/block_service_spec.rb
+++ b/spec/services/block_service_spec.rb
@@ -28,13 +28,6 @@ RSpec.describe BlockService, type: :service do
     it 'creates a blocking relation' do
       expect(sender.blocking?(bob)).to be true
     end
-
-    it 'sends a block salmon slap' do
-      expect(a_request(:post, "http://salmon.example.com/").with { |req|
-        xml = OStatus2::Salmon.new.unpack(req.body)
-        xml.match(OStatus::TagManager::VERBS[:block])
-      }).to have_been_made.once
-    end
   end
 
   describe 'remote ActivityPub' do
diff --git a/spec/services/favourite_service_spec.rb b/spec/services/favourite_service_spec.rb
index 0a20ccf6e..4c29ea77b 100644
--- a/spec/services/favourite_service_spec.rb
+++ b/spec/services/favourite_service_spec.rb
@@ -30,13 +30,6 @@ RSpec.describe FavouriteService, type: :service do
     it 'creates a favourite' do
       expect(status.favourites.first).to_not be_nil
     end
-
-    it 'sends a salmon slap' do
-      expect(a_request(:post, "http://salmon.example.com/").with { |req|
-        xml = OStatus2::Salmon.new.unpack(req.body)
-        xml.match(OStatus::TagManager::VERBS[:favorite])
-      }).to have_been_made.once
-    end
   end
 
   describe 'remote ActivityPub' do
diff --git a/spec/services/fetch_remote_account_service_spec.rb b/spec/services/fetch_remote_account_service_spec.rb
index 3cd86708b..37e9910d4 100644
--- a/spec/services/fetch_remote_account_service_spec.rb
+++ b/spec/services/fetch_remote_account_service_spec.rb
@@ -36,36 +36,6 @@ RSpec.describe FetchRemoteAccountService, type: :service do
     include_examples 'return Account'
   end
 
-  context 'protocol is :ostatus' do
-    let(:prefetched_body) { xml }
-    let(:protocol) { :ostatus }
-
-    before do
-      stub_request(:get, "https://kickass.zone/.well-known/webfinger?resource=acct:localhost@kickass.zone").to_return(request_fixture('webfinger-hacker3.txt'))
-      stub_request(:get, "https://kickass.zone/api/statuses/user_timeline/7477.atom").to_return(request_fixture('feed.txt'))
-    end
-
-    include_examples 'return Account'
-
-    it 'does not update account information if XML comes from an unverified domain' do
-      feed_xml = <<-XML.squish
-        <?xml version="1.0" encoding="UTF-8"?>
-        <feed xml:lang="en-US" xmlns="http://www.w3.org/2005/Atom" xmlns:thr="http://purl.org/syndication/thread/1.0" xmlns:georss="http://www.georss.org/georss" xmlns:activity="http://activitystrea.ms/spec/1.0/" xmlns:media="http://purl.org/syndication/atommedia" xmlns:poco="http://portablecontacts.net/spec/1.0" xmlns:ostatus="http://ostatus.org/schema/1.0" xmlns:statusnet="http://status.net/schema/api/1/">
-          <author>
-            <activity:object-type>http://activitystrea.ms/schema/1.0/person</activity:object-type>
-            <uri>http://kickass.zone/users/localhost</uri>
-            <name>localhost</name>
-            <poco:preferredUsername>localhost</poco:preferredUsername>
-            <poco:displayName>Villain!!!</poco:displayName>
-          </author>
-        </feed>
-      XML
-
-      returned_account = described_class.new.call('https://real-fake-domains.com/alice', feed_xml, :ostatus)
-      expect(returned_account.display_name).to_not eq 'Villain!!!'
-    end
-  end
-
   context 'when prefetched_body is nil' do
     context 'protocol is :activitypub' do
       before do
@@ -75,15 +45,5 @@ RSpec.describe FetchRemoteAccountService, type: :service do
 
       include_examples 'return Account'
     end
-
-    context 'protocol is :ostatus' do
-      before do
-        stub_request(:get, url).to_return(status: 200, body: xml, headers: { 'Content-Type' => 'application/atom+xml' })
-        stub_request(:get, "https://kickass.zone/.well-known/webfinger?resource=acct:localhost@kickass.zone").to_return(request_fixture('webfinger-hacker3.txt'))
-        stub_request(:get, "https://kickass.zone/api/statuses/user_timeline/7477.atom").to_return(request_fixture('feed.txt'))
-      end
-
-      include_examples 'return Account'
-    end
   end
 end
diff --git a/spec/services/follow_service_spec.rb b/spec/services/follow_service_spec.rb
index 3c4ec59be..86c85293e 100644
--- a/spec/services/follow_service_spec.rb
+++ b/spec/services/follow_service_spec.rb
@@ -96,74 +96,6 @@ RSpec.describe FollowService, type: :service do
     end
   end
 
-  context 'remote OStatus account' do
-    describe 'locked account' do
-      let(:bob) { Fabricate(:user, email: 'bob@example.com', account: Fabricate(:account, protocol: :ostatus, locked: true, username: 'bob', domain: 'example.com', salmon_url: 'http://salmon.example.com')).account }
-
-      before do
-        stub_request(:post, "http://salmon.example.com/").to_return(:status => 200, :body => "", :headers => {})
-        subject.call(sender, bob.acct)
-      end
-
-      it 'creates a follow request' do
-        expect(FollowRequest.find_by(account: sender, target_account: bob)).to_not be_nil
-      end
-
-      it 'sends a follow request salmon slap' do
-        expect(a_request(:post, "http://salmon.example.com/").with { |req|
-          xml = OStatus2::Salmon.new.unpack(req.body)
-          xml.match(OStatus::TagManager::VERBS[:request_friend])
-        }).to have_been_made.once
-      end
-    end
-
-    describe 'unlocked account' do
-      let(:bob) { Fabricate(:user, email: 'bob@example.com', account: Fabricate(:account, protocol: :ostatus, username: 'bob', domain: 'example.com', salmon_url: 'http://salmon.example.com', hub_url: 'http://hub.example.com')).account }
-
-      before do
-        stub_request(:post, "http://salmon.example.com/").to_return(:status => 200, :body => "", :headers => {})
-        stub_request(:post, "http://hub.example.com/").to_return(status: 202)
-        subject.call(sender, bob.acct)
-      end
-
-      it 'creates a following relation' do
-        expect(sender.following?(bob)).to be true
-      end
-
-      it 'sends a follow salmon slap' do
-        expect(a_request(:post, "http://salmon.example.com/").with { |req|
-          xml = OStatus2::Salmon.new.unpack(req.body)
-          xml.match(OStatus::TagManager::VERBS[:follow])
-        }).to have_been_made.once
-      end
-
-      it 'subscribes to PuSH' do
-        expect(a_request(:post, "http://hub.example.com/")).to have_been_made.once
-      end
-    end
-
-    describe 'already followed account' do
-      let(:bob) { Fabricate(:user, email: 'bob@example.com', account: Fabricate(:account, protocol: :ostatus, username: 'bob', domain: 'example.com', salmon_url: 'http://salmon.example.com', hub_url: 'http://hub.example.com')).account }
-
-      before do
-        sender.follow!(bob)
-        subject.call(sender, bob.acct)
-      end
-
-      it 'keeps a following relation' do
-        expect(sender.following?(bob)).to be true
-      end
-
-      it 'does not send a follow salmon slap' do
-        expect(a_request(:post, "http://salmon.example.com/")).not_to have_been_made
-      end
-
-      it 'does not subscribe to PuSH' do
-        expect(a_request(:post, "http://hub.example.com/")).not_to have_been_made
-      end
-    end
-  end
-
   context 'remote ActivityPub account' do
     let(:bob) { Fabricate(:user, account: Fabricate(:account, username: 'bob', domain: 'example.com', protocol: :activitypub, inbox_url: 'http://example.com/inbox')).account }
 
diff --git a/spec/services/import_service_spec.rb b/spec/services/import_service_spec.rb
index 5cf2dadf0..5355133f4 100644
--- a/spec/services/import_service_spec.rb
+++ b/spec/services/import_service_spec.rb
@@ -3,7 +3,11 @@ require 'rails_helper'
 RSpec.describe ImportService, type: :service do
   let!(:account) { Fabricate(:account, locked: false) }
   let!(:bob)     { Fabricate(:account, username: 'bob', locked: false) }
-  let!(:eve)     { Fabricate(:account, username: 'eve', domain: 'example.com', locked: false) }
+  let!(:eve)     { Fabricate(:account, username: 'eve', domain: 'example.com', locked: false, protocol: :activitypub, inbox_url: 'https://example.com/inbox') }
+
+  before do
+    stub_request(:post, "https://example.com/inbox").to_return(status: 200)
+  end
 
   context 'import old-style list of muted users' do
     subject { ImportService.new }
@@ -95,7 +99,8 @@ RSpec.describe ImportService, type: :service do
       let(:import) { Import.create(account: account, type: 'following', data: csv) }
       it 'follows the listed accounts, including boosts' do
         subject.call(import)
-        expect(account.following.count).to eq 2
+        expect(account.following.count).to eq 1
+        expect(account.follow_requests.count).to eq 1
         expect(Follow.find_by(account: account, target_account: bob).show_reblogs).to be true
       end
     end
@@ -106,7 +111,8 @@ RSpec.describe ImportService, type: :service do
       it 'follows the listed accounts, including notifications' do
         account.follow!(bob, reblogs: false)
         subject.call(import)
-        expect(account.following.count).to eq 2
+        expect(account.following.count).to eq 1
+        expect(account.follow_requests.count).to eq 1
         expect(Follow.find_by(account: account, target_account: bob).show_reblogs).to be true
       end
     end
@@ -117,7 +123,8 @@ RSpec.describe ImportService, type: :service do
       it 'mutes the listed accounts, including notifications' do
         account.follow!(bob, reblogs: false)
         subject.call(import)
-        expect(account.following.count).to eq 2
+        expect(account.following.count).to eq 1
+        expect(account.follow_requests.count).to eq 1
         expect(Follow.find_by(account: account, target_account: bob).show_reblogs).to be true
       end
     end
@@ -136,9 +143,10 @@ RSpec.describe ImportService, type: :service do
       let(:import) { Import.create(account: account, type: 'following', data: csv) }
       it 'follows the listed accounts, respecting boosts' do
         subject.call(import)
-        expect(account.following.count).to eq 2
+        expect(account.following.count).to eq 1
+        expect(account.follow_requests.count).to eq 1
         expect(Follow.find_by(account: account, target_account: bob).show_reblogs).to be true
-        expect(Follow.find_by(account: account, target_account: eve).show_reblogs).to be false
+        expect(FollowRequest.find_by(account: account, target_account: eve).show_reblogs).to be false
       end
     end
 
@@ -148,9 +156,10 @@ RSpec.describe ImportService, type: :service do
       it 'mutes the listed accounts, respecting notifications' do
         account.follow!(bob, reblogs: true)
         subject.call(import)
-        expect(account.following.count).to eq 2
+        expect(account.following.count).to eq 1
+        expect(account.follow_requests.count).to eq 1
         expect(Follow.find_by(account: account, target_account: bob).show_reblogs).to be true
-        expect(Follow.find_by(account: account, target_account: eve).show_reblogs).to be false
+        expect(FollowRequest.find_by(account: account, target_account: eve).show_reblogs).to be false
       end
     end
 
@@ -160,9 +169,10 @@ RSpec.describe ImportService, type: :service do
       it 'mutes the listed accounts, respecting notifications' do
         account.follow!(bob, reblogs: true)
         subject.call(import)
-        expect(account.following.count).to eq 2
+        expect(account.following.count).to eq 1
+        expect(account.follow_requests.count).to eq 1
         expect(Follow.find_by(account: account, target_account: bob).show_reblogs).to be true
-        expect(Follow.find_by(account: account, target_account: eve).show_reblogs).to be false
+        expect(FollowRequest.find_by(account: account, target_account: eve).show_reblogs).to be false
       end
     end
   end
diff --git a/spec/services/post_status_service_spec.rb b/spec/services/post_status_service_spec.rb
index facbe977f..bf06f50e9 100644
--- a/spec/services/post_status_service_spec.rb
+++ b/spec/services/post_status_service_spec.rb
@@ -144,7 +144,6 @@ RSpec.describe PostStatusService, type: :service do
 
   it 'gets distributed' do
     allow(DistributionWorker).to receive(:perform_async)
-    allow(Pubsubhubbub::DistributionWorker).to receive(:perform_async)
     allow(ActivityPub::DistributionWorker).to receive(:perform_async)
 
     account = Fabricate(:account)
@@ -152,7 +151,6 @@ RSpec.describe PostStatusService, type: :service do
     status = subject.call(account, text: "test status update")
 
     expect(DistributionWorker).to have_received(:perform_async).with(status.id)
-    expect(Pubsubhubbub::DistributionWorker).to have_received(:perform_async).with(status.stream_entry.id)
     expect(ActivityPub::DistributionWorker).to have_received(:perform_async).with(status.id)
   end
 
diff --git a/spec/services/process_feed_service_spec.rb b/spec/services/process_feed_service_spec.rb
deleted file mode 100644
index 9d3465f3f..000000000
--- a/spec/services/process_feed_service_spec.rb
+++ /dev/null
@@ -1,252 +0,0 @@
-require 'rails_helper'
-
-RSpec.describe ProcessFeedService, type: :service do
-  subject { ProcessFeedService.new }
-
-  describe 'processing a feed' do
-    let(:body) { File.read(Rails.root.join('spec', 'fixtures', 'xml', 'mastodon.atom')) }
-    let(:account) { Fabricate(:account, username: 'localhost', domain: 'kickass.zone') }
-
-    before do
-      stub_request(:post, "https://pubsubhubbub.superfeedr.com/").to_return(:status => 200, :body => "", :headers => {})
-      stub_request(:head, "http://kickass.zone/media/2").to_return(:status => 404)
-      stub_request(:head, "http://kickass.zone/media/3").to_return(:status => 404)
-      stub_request(:get, "http://kickass.zone/system/accounts/avatars/000/000/001/large/eris.png").to_return(request_fixture('avatar.txt'))
-      stub_request(:get, "http://kickass.zone/system/media_attachments/files/000/000/002/original/morpheus_linux.jpg?1476059910").to_return(request_fixture('attachment1.txt'))
-      stub_request(:get, "http://kickass.zone/system/media_attachments/files/000/000/003/original/gizmo.jpg?1476060065").to_return(request_fixture('attachment2.txt'))
-    end
-
-    context 'when domain does not reject media' do
-      before do
-        subject.call(body, account)
-      end
-
-      it 'updates remote user\'s account information' do
-        account.reload
-        expect(account.display_name).to eq '::1'
-        expect(account).to have_attached_file(:avatar)
-        expect(account.avatar_file_name).not_to be_nil
-      end
-
-      it 'creates posts' do
-        expect(Status.find_by(uri: 'tag:kickass.zone,2016-10-10:objectId=1:objectType=Status')).to_not be_nil
-        expect(Status.find_by(uri: 'tag:kickass.zone,2016-10-10:objectId=2:objectType=Status')).to_not be_nil
-      end
-
-      it 'marks replies as replies' do
-        status = Status.find_by(uri: 'tag:kickass.zone,2016-10-10:objectId=2:objectType=Status')
-        expect(status.reply?).to be true
-      end
-
-      it 'sets account being replied to when possible' do
-        status = Status.find_by(uri: 'tag:kickass.zone,2016-10-10:objectId=2:objectType=Status')
-        expect(status.in_reply_to_account_id).to eq status.account_id
-      end
-
-      it 'ignores delete statuses unless they existed before' do
-        expect(Status.find_by(uri: 'tag:kickass.zone,2016-10-10:objectId=3:objectType=Status')).to be_nil
-        expect(Status.find_by(uri: 'tag:kickass.zone,2016-10-10:objectId=12:objectType=Status')).to be_nil
-      end
-
-      it 'does not create statuses for follows' do
-        expect(Status.find_by(uri: 'tag:kickass.zone,2016-10-10:objectId=1:objectType=Follow')).to be_nil
-        expect(Status.find_by(uri: 'tag:kickass.zone,2016-10-10:objectId=2:objectType=Follow')).to be_nil
-        expect(Status.find_by(uri: 'tag:kickass.zone,2016-10-10:objectId=4:objectType=Follow')).to be_nil
-        expect(Status.find_by(uri: 'tag:kickass.zone,2016-10-10:objectId=7:objectType=Follow')).to be_nil
-      end
-
-      it 'does not create statuses for favourites' do
-        expect(Status.find_by(uri: 'tag:kickass.zone,2016-10-10:objectId=2:objectType=Favourite')).to be_nil
-        expect(Status.find_by(uri: 'tag:kickass.zone,2016-10-10:objectId=3:objectType=Favourite')).to be_nil
-      end
-
-      it 'creates posts with media' do
-        status = Status.find_by(uri: 'tag:kickass.zone,2016-10-10:objectId=14:objectType=Status')
-
-        expect(status).to_not be_nil
-        expect(status.media_attachments.first).to have_attached_file(:file)
-        expect(status.media_attachments.first.image?).to be true
-        expect(status.media_attachments.first.file_file_name).not_to be_nil
-      end
-    end
-
-    context 'when domain is set to reject media' do
-      let!(:domain_block) { Fabricate(:domain_block, domain: 'kickass.zone', reject_media: true) }
-
-      before do
-        subject.call(body, account)
-      end
-
-      it 'updates remote user\'s account information' do
-        account.reload
-        expect(account.display_name).to eq '::1'
-      end
-
-      it 'rejects remote user\'s avatar' do
-        account.reload
-        expect(account.display_name).to eq '::1'
-        expect(account.avatar_file_name).to be_nil
-      end
-
-      it 'creates posts' do
-        expect(Status.find_by(uri: 'tag:kickass.zone,2016-10-10:objectId=1:objectType=Status')).to_not be_nil
-        expect(Status.find_by(uri: 'tag:kickass.zone,2016-10-10:objectId=2:objectType=Status')).to_not be_nil
-      end
-
-      it 'creates posts with remote-only media' do
-        status = Status.find_by(uri: 'tag:kickass.zone,2016-10-10:objectId=14:objectType=Status')
-
-        expect(status).to_not be_nil
-        expect(status.media_attachments.first.file_file_name).to be_nil
-        expect(status.media_attachments.first.unknown?).to be true
-      end
-    end
-  end
-
-  it 'does not accept tampered reblogs' do
-    good_actor = Fabricate(:account, username: 'tracer', domain: 'overwatch.com')
-
-    real_body = <<XML
-<?xml version="1.0"?>
-<entry xmlns="http://www.w3.org/2005/Atom" xmlns:thr="http://purl.org/syndication/thread/1.0" xmlns:activity="http://activitystrea.ms/spec/1.0/" xmlns:poco="http://portablecontacts.net/spec/1.0" xmlns:media="http://purl.org/syndication/atommedia" xmlns:ostatus="http://ostatus.org/schema/1.0" xmlns:mastodon="http://mastodon.social/schema/1.0">
-  <id>tag:overwatch.com,2017-04-27:objectId=4467137:objectType=Status</id>
-  <published>2017-04-27T13:49:25Z</published>
-  <updated>2017-04-27T13:49:25Z</updated>
-  <activity:object-type>http://activitystrea.ms/schema/1.0/note</activity:object-type>
-  <activity:verb>http://activitystrea.ms/schema/1.0/post</activity:verb>
-  <author>
-    <id>https://overwatch.com/users/tracer</id>
-    <activity:object-type>http://activitystrea.ms/schema/1.0/person</activity:object-type>
-    <uri>https://overwatch.com/users/tracer</uri>
-    <name>tracer</name>
-  </author>
-  <content type="html">Overwatch rocks</content>
-</entry>
-XML
-
-    stub_request(:get, 'https://overwatch.com/users/tracer/updates/1').to_return(status: 200, body: real_body, headers: { 'Content-Type' => 'application/atom+xml' })
-
-    bad_actor = Fabricate(:account, username: 'sombra', domain: 'talon.xyz')
-
-    body = <<XML
-<?xml version="1.0"?>
-<entry xmlns="http://www.w3.org/2005/Atom" xmlns:thr="http://purl.org/syndication/thread/1.0" xmlns:activity="http://activitystrea.ms/spec/1.0/" xmlns:poco="http://portablecontacts.net/spec/1.0" xmlns:media="http://purl.org/syndication/atommedia" xmlns:ostatus="http://ostatus.org/schema/1.0" xmlns:mastodon="http://mastodon.social/schema/1.0">
-  <id>tag:talon.xyz,2017-04-27:objectId=4467137:objectType=Status</id>
-  <published>2017-04-27T13:49:25Z</published>
-  <updated>2017-04-27T13:49:25Z</updated>
-  <author>
-    <id>https://talon.xyz/users/sombra</id>
-    <activity:object-type>http://activitystrea.ms/schema/1.0/person</activity:object-type>
-    <uri>https://talon.xyz/users/sombra</uri>
-    <name>sombra</name>
-  </author>
-  <activity:object-type>http://activitystrea.ms/schema/1.0/activity</activity:object-type>
-  <activity:verb>http://activitystrea.ms/schema/1.0/share</activity:verb>
-  <content type="html">Overwatch SUCKS AHAHA</content>
-  <activity:object>
-    <id>tag:overwatch.com,2017-04-27:objectId=4467137:objectType=Status</id>
-    <activity:object-type>http://activitystrea.ms/schema/1.0/note</activity:object-type>
-    <activity:verb>http://activitystrea.ms/schema/1.0/post</activity:verb>
-    <author>
-      <id>https://overwatch.com/users/tracer</id>
-      <activity:object-type>http://activitystrea.ms/schema/1.0/person</activity:object-type>
-      <uri>https://overwatch.com/users/tracer</uri>
-      <name>tracer</name>
-    </author>
-    <content type="html">Overwatch SUCKS AHAHA</content>
-    <link rel="alternate" type="text/html" href="https://overwatch.com/users/tracer/updates/1" />
-  </activity:object>
-</entry>
-XML
-    created_statuses = subject.call(body, bad_actor)
-
-    expect(created_statuses.first.reblog?).to be true
-    expect(created_statuses.first.account_id).to eq bad_actor.id
-    expect(created_statuses.first.reblog.account_id).to eq good_actor.id
-    expect(created_statuses.first.reblog.text).to eq 'Overwatch rocks'
-  end
-
-  it 'ignores reblogs if it failed to retrieve reblogged statuses' do
-    stub_request(:get, 'https://overwatch.com/users/tracer/updates/1').to_return(status: 404)
-
-    actor = Fabricate(:account, username: 'tracer', domain: 'overwatch.com')
-
-    body = <<XML
-<?xml version="1.0"?>
-<entry xmlns="http://www.w3.org/2005/Atom" xmlns:thr="http://purl.org/syndication/thread/1.0" xmlns:activity="http://activitystrea.ms/spec/1.0/" xmlns:poco="http://portablecontacts.net/spec/1.0" xmlns:media="http://purl.org/syndication/atommedia" xmlns:ostatus="http://ostatus.org/schema/1.0" xmlns:mastodon="http://mastodon.social/schema/1.0">
-  <id>tag:overwatch.com,2017-04-27:objectId=4467137:objectType=Status</id>
-  <published>2017-04-27T13:49:25Z</published>
-  <updated>2017-04-27T13:49:25Z</updated>
-  <author>
-    <id>https://overwatch.com/users/tracer</id>
-    <activity:object-type>http://activitystrea.ms/schema/1.0/person</activity:object-type>
-    <uri>https://overwatch.com/users/tracer</uri>
-    <name>tracer</name>
-  </author>
-  <activity:object-type>http://activitystrea.ms/schema/1.0/activity</activity:object-type>
-  <activity:verb>http://activitystrea.ms/schema/1.0/share</activity:verb>
-  <content type="html">Overwatch rocks</content>
-  <activity:object>
-    <id>tag:overwatch.com,2017-04-27:objectId=4467137:objectType=Status</id>
-    <activity:object-type>http://activitystrea.ms/schema/1.0/note</activity:object-type>
-    <activity:verb>http://activitystrea.ms/schema/1.0/post</activity:verb>
-    <author>
-      <id>https://overwatch.com/users/tracer</id>
-      <activity:object-type>http://activitystrea.ms/schema/1.0/person</activity:object-type>
-      <uri>https://overwatch.com/users/tracer</uri>
-      <name>tracer</name>
-    </author>
-    <content type="html">Overwatch rocks</content>
-    <link rel="alternate" type="text/html" href="https://overwatch.com/users/tracer/updates/1" />
-  </activity:object>
-XML
-
-    created_statuses = subject.call(body, actor)
-
-    expect(created_statuses).to eq []
-  end
-
-  it 'ignores statuses with an out-of-order delete' do
-    sender = Fabricate(:account, username: 'tracer', domain: 'overwatch.com')
-
-    delete_body = <<XML
-<?xml version="1.0"?>
-<entry xmlns="http://www.w3.org/2005/Atom" xmlns:thr="http://purl.org/syndication/thread/1.0" xmlns:activity="http://activitystrea.ms/spec/1.0/" xmlns:poco="http://portablecontacts.net/spec/1.0" xmlns:media="http://purl.org/syndication/atommedia" xmlns:ostatus="http://ostatus.org/schema/1.0" xmlns:mastodon="http://mastodon.social/schema/1.0">
-  <id>tag:overwatch.com,2017-04-27:objectId=4487555:objectType=Status</id>
-  <published>2017-04-27T13:49:25Z</published>
-  <updated>2017-04-27T13:49:25Z</updated>
-  <activity:object-type>http://activitystrea.ms/schema/1.0/note</activity:object-type>
-  <activity:verb>http://activitystrea.ms/schema/1.0/delete</activity:verb>
-  <author>
-    <id>https://overwatch.com/users/tracer</id>
-    <activity:object-type>http://activitystrea.ms/schema/1.0/person</activity:object-type>
-    <uri>https://overwatch.com/users/tracer</uri>
-    <name>tracer</name>
-  </author>
-</entry>
-XML
-
-    status_body = <<XML
-<?xml version="1.0"?>
-<entry xmlns="http://www.w3.org/2005/Atom" xmlns:thr="http://purl.org/syndication/thread/1.0" xmlns:activity="http://activitystrea.ms/spec/1.0/" xmlns:poco="http://portablecontacts.net/spec/1.0" xmlns:media="http://purl.org/syndication/atommedia" xmlns:ostatus="http://ostatus.org/schema/1.0" xmlns:mastodon="http://mastodon.social/schema/1.0">
-  <id>tag:overwatch.com,2017-04-27:objectId=4487555:objectType=Status</id>
-  <published>2017-04-27T13:49:25Z</published>
-  <updated>2017-04-27T13:49:25Z</updated>
-  <activity:object-type>http://activitystrea.ms/schema/1.0/note</activity:object-type>
-  <activity:verb>http://activitystrea.ms/schema/1.0/post</activity:verb>
-  <author>
-    <id>https://overwatch.com/users/tracer</id>
-    <activity:object-type>http://activitystrea.ms/schema/1.0/person</activity:object-type>
-    <uri>https://overwatch.com/users/tracer</uri>
-    <name>tracer</name>
-  </author>
-  <content type="html">Overwatch rocks</content>
-</entry>
-XML
-
-    subject.call(delete_body, sender)
-    created_statuses = subject.call(status_body, sender)
-
-    expect(created_statuses).to be_empty
-  end
-end
diff --git a/spec/services/process_interaction_service_spec.rb b/spec/services/process_interaction_service_spec.rb
deleted file mode 100644
index b858c19d0..000000000
--- a/spec/services/process_interaction_service_spec.rb
+++ /dev/null
@@ -1,151 +0,0 @@
-require 'rails_helper'
-
-RSpec.describe ProcessInteractionService, type: :service do
-  let(:receiver) { Fabricate(:user, email: 'alice@example.com', account: Fabricate(:account, username: 'alice')).account }
-  let(:sender)   { Fabricate(:user, email: 'bob@example.com', account: Fabricate(:account, username: 'bob')).account }
-  let(:remote_sender) { Fabricate(:account, username: 'carol', domain: 'localdomain.com', uri: 'https://webdomain.com/users/carol') }
-
-  subject { ProcessInteractionService.new }
-
-  describe 'status delete slap' do
-    let(:remote_status) { Fabricate(:status, account: remote_sender) }
-    let(:envelope) { OStatus2::Salmon.new.pack(payload, sender.keypair) }
-    let(:payload) {
-      <<~XML
-        <entry xmlns="http://www.w3.org/2005/Atom" xmlns:activity="http://activitystrea.ms/spec/1.0/">
-          <author>
-            <email>carol@localdomain.com</email>
-            <name>carol</name>
-            <uri>https://webdomain.com/users/carol</uri>
-          </author>
-
-          <id>#{remote_status.id}</id>
-          <activity:verb>http://activitystrea.ms/schema/1.0/delete</activity:verb>
-        </entry>
-      XML
-    }
-
-    before do
-      receiver.update(locked: true)
-      remote_sender.update(private_key: sender.private_key, public_key: remote_sender.public_key)
-    end
-
-    it 'deletes a record' do
-      expect(RemovalWorker).to receive(:perform_async).with(remote_status.id)
-      subject.call(envelope, receiver)
-    end
-  end
-
-  describe 'follow request slap' do
-    before do
-      receiver.update(locked: true)
-
-      payload = <<XML
-<entry xmlns="http://www.w3.org/2005/Atom" xmlns:activity="http://activitystrea.ms/spec/1.0/">
-  <author>
-    <name>bob</name>
-    <uri>https://cb6e6126.ngrok.io/users/bob</uri>
-  </author>
-
-  <id>someIdHere</id>
-  <activity:verb>http://activitystrea.ms/schema/1.0/request-friend</activity:verb>
-</entry>
-XML
-
-      envelope = OStatus2::Salmon.new.pack(payload, sender.keypair)
-      subject.call(envelope, receiver)
-    end
-
-    it 'creates a record' do
-      expect(FollowRequest.find_by(account: sender, target_account: receiver)).to_not be_nil
-    end
-  end
-
-  describe 'follow request slap from known remote user identified by email' do
-    before do
-      receiver.update(locked: true)
-      # Copy already-generated key
-      remote_sender.update(private_key: sender.private_key, public_key: remote_sender.public_key)
-
-      payload = <<XML
-<entry xmlns="http://www.w3.org/2005/Atom" xmlns:activity="http://activitystrea.ms/spec/1.0/">
-  <author>
-    <email>carol@localdomain.com</email>
-    <name>carol</name>
-    <uri>https://webdomain.com/users/carol</uri>
-  </author>
-
-  <id>someIdHere</id>
-  <activity:verb>http://activitystrea.ms/schema/1.0/request-friend</activity:verb>
-</entry>
-XML
-
-      envelope = OStatus2::Salmon.new.pack(payload, remote_sender.keypair)
-      subject.call(envelope, receiver)
-    end
-
-    it 'creates a record' do
-      expect(FollowRequest.find_by(account: remote_sender, target_account: receiver)).to_not be_nil
-    end
-  end
-
-  describe 'follow request authorization slap' do
-    before do
-      receiver.update(locked: true)
-      FollowRequest.create(account: sender, target_account: receiver)
-
-      payload = <<XML
-<entry xmlns="http://www.w3.org/2005/Atom" xmlns:activity="http://activitystrea.ms/spec/1.0/">
-  <author>
-    <name>alice</name>
-    <uri>https://cb6e6126.ngrok.io/users/alice</uri>
-  </author>
-
-  <id>someIdHere</id>
-  <activity:verb>http://activitystrea.ms/schema/1.0/authorize</activity:verb>
-</entry>
-XML
-
-      envelope = OStatus2::Salmon.new.pack(payload, receiver.keypair)
-      subject.call(envelope, sender)
-    end
-
-    it 'creates a follow relationship' do
-      expect(Follow.find_by(account: sender, target_account: receiver)).to_not be_nil
-    end
-
-    it 'removes the follow request' do
-      expect(FollowRequest.find_by(account: sender, target_account: receiver)).to be_nil
-    end
-  end
-
-  describe 'follow request rejection slap' do
-    before do
-      receiver.update(locked: true)
-      FollowRequest.create(account: sender, target_account: receiver)
-
-      payload = <<XML
-<entry xmlns="http://www.w3.org/2005/Atom" xmlns:activity="http://activitystrea.ms/spec/1.0/">
-  <author>
-    <name>alice</name>
-    <uri>https://cb6e6126.ngrok.io/users/alice</uri>
-  </author>
-
-  <id>someIdHere</id>
-  <activity:verb>http://activitystrea.ms/schema/1.0/reject</activity:verb>
-</entry>
-XML
-
-      envelope = OStatus2::Salmon.new.pack(payload, receiver.keypair)
-      subject.call(envelope, sender)
-    end
-
-    it 'does not create a follow relationship' do
-      expect(Follow.find_by(account: sender, target_account: receiver)).to be_nil
-    end
-
-    it 'removes the follow request' do
-      expect(FollowRequest.find_by(account: sender, target_account: receiver)).to be_nil
-    end
-  end
-end
diff --git a/spec/services/process_mentions_service_spec.rb b/spec/services/process_mentions_service_spec.rb
index 8a6bb44ac..35a804f2b 100644
--- a/spec/services/process_mentions_service_spec.rb
+++ b/spec/services/process_mentions_service_spec.rb
@@ -18,10 +18,6 @@ RSpec.describe ProcessMentionsService, type: :service do
     it 'creates a mention' do
       expect(remote_user.mentions.where(status: status).count).to eq 1
     end
-
-    it 'posts to remote user\'s Salmon end point' do
-      expect(a_request(:post, remote_user.salmon_url)).to have_been_made.once
-    end
   end
 
   context 'OStatus with private toot' do
diff --git a/spec/services/pubsubhubbub/subscribe_service_spec.rb b/spec/services/pubsubhubbub/subscribe_service_spec.rb
deleted file mode 100644
index 01c956230..000000000
--- a/spec/services/pubsubhubbub/subscribe_service_spec.rb
+++ /dev/null
@@ -1,71 +0,0 @@
-# frozen_string_literal: true
-
-require 'rails_helper'
-
-describe Pubsubhubbub::SubscribeService, type: :service do
-  describe '#call' do
-    subject { described_class.new }
-    let(:user_account) { Fabricate(:account) }
-
-    context 'with a nil account' do
-      it 'returns the invalid topic status results' do
-        result = service_call(account: nil)
-
-        expect(result).to eq invalid_topic_status
-      end
-    end
-
-    context 'with an invalid callback url' do
-      it 'returns invalid callback status when callback is blank' do
-        result = service_call(callback: '')
-
-        expect(result).to eq invalid_callback_status
-      end
-      it 'returns invalid callback status when callback is not a URI' do
-        result = service_call(callback: 'invalid-hostname')
-
-        expect(result).to eq invalid_callback_status
-      end
-    end
-
-    context 'with a blocked domain in the callback' do
-      it 'returns callback not allowed' do
-        Fabricate(:domain_block, domain: 'test.host', severity: :suspend)
-        result = service_call(callback: 'https://test.host/api')
-
-        expect(result).to eq not_allowed_callback_status
-      end
-    end
-
-    context 'with a valid account and callback' do
-      it 'returns success status and confirms subscription' do
-        allow(Pubsubhubbub::ConfirmationWorker).to receive(:perform_async).and_return(nil)
-        subscription = Fabricate(:subscription, account: user_account)
-
-        result = service_call(callback: subscription.callback_url)
-        expect(result).to eq success_status
-        expect(Pubsubhubbub::ConfirmationWorker).to have_received(:perform_async).with(subscription.id, 'subscribe', 'asdf', 3600)
-      end
-    end
-  end
-
-  def service_call(account: user_account, callback: 'https://callback.host', secret: 'asdf', lease_seconds: 3600)
-    subject.call(account, callback, secret, lease_seconds)
-  end
-
-  def invalid_topic_status
-    ['Invalid topic URL', 422]
-  end
-
-  def invalid_callback_status
-    ['Invalid callback URL', 422]
-  end
-
-  def not_allowed_callback_status
-    ['Callback URL not allowed', 403]
-  end
-
-  def success_status
-    ['', 202]
-  end
-end
diff --git a/spec/services/pubsubhubbub/unsubscribe_service_spec.rb b/spec/services/pubsubhubbub/unsubscribe_service_spec.rb
deleted file mode 100644
index 7ed9fc5af..000000000
--- a/spec/services/pubsubhubbub/unsubscribe_service_spec.rb
+++ /dev/null
@@ -1,46 +0,0 @@
-# frozen_string_literal: true
-
-require 'rails_helper'
-
-describe Pubsubhubbub::UnsubscribeService, type: :service do
-  describe '#call' do
-    subject { described_class.new }
-
-    context 'with a nil account' do
-      it 'returns an invalid topic status' do
-        result = subject.call(nil, 'callback.host')
-
-        expect(result).to eq invalid_topic_status
-      end
-    end
-
-    context 'with a valid account' do
-      let(:account) { Fabricate(:account) }
-
-      it 'returns a valid topic status and does not run confirm when no subscription' do
-        allow(Pubsubhubbub::ConfirmationWorker).to receive(:perform_async).and_return(nil)
-        result = subject.call(account, 'callback.host')
-
-        expect(result).to eq valid_topic_status
-        expect(Pubsubhubbub::ConfirmationWorker).not_to have_received(:perform_async)
-      end
-
-      it 'returns a valid topic status and does run confirm when there is a subscription' do
-        subscription = Fabricate(:subscription, account: account, callback_url: 'callback.host')
-        allow(Pubsubhubbub::ConfirmationWorker).to receive(:perform_async).and_return(nil)
-        result = subject.call(account, 'callback.host')
-
-        expect(result).to eq valid_topic_status
-        expect(Pubsubhubbub::ConfirmationWorker).to have_received(:perform_async).with(subscription.id, 'unsubscribe')
-      end
-    end
-
-    def invalid_topic_status
-      ['Invalid topic URL', 422]
-    end
-
-    def valid_topic_status
-      ['', 202]
-    end
-  end
-end
diff --git a/spec/services/reblog_service_spec.rb b/spec/services/reblog_service_spec.rb
index 9d84c41d5..58fb46f0f 100644
--- a/spec/services/reblog_service_spec.rb
+++ b/spec/services/reblog_service_spec.rb
@@ -46,10 +46,6 @@ RSpec.describe ReblogService, type: :service do
     it 'creates a reblog' do
       expect(status.reblogs.count).to eq 1
     end
-
-    it 'sends a Salmon slap for a remote reblog' do
-      expect(a_request(:post, 'http://salmon.example.com')).to have_been_made
-    end
   end
 
   context 'ActivityPub' do
diff --git a/spec/services/reject_follow_service_spec.rb b/spec/services/reject_follow_service_spec.rb
index e5ac37ed9..1aec060db 100644
--- a/spec/services/reject_follow_service_spec.rb
+++ b/spec/services/reject_follow_service_spec.rb
@@ -38,13 +38,6 @@ RSpec.describe RejectFollowService, type: :service do
     it 'does not create follow relation' do
       expect(bob.following?(sender)).to be false
     end
-
-    it 'sends a follow request rejection salmon slap' do
-      expect(a_request(:post, "http://salmon.example.com/").with { |req|
-        xml = OStatus2::Salmon.new.unpack(req.body)
-        xml.match(OStatus::TagManager::VERBS[:reject])
-      }).to have_been_made.once
-    end
   end
 
   describe 'remote ActivityPub' do
diff --git a/spec/services/remove_status_service_spec.rb b/spec/services/remove_status_service_spec.rb
index 7bba83a60..48191d47c 100644
--- a/spec/services/remove_status_service_spec.rb
+++ b/spec/services/remove_status_service_spec.rb
@@ -32,23 +32,10 @@ RSpec.describe RemoveStatusService, type: :service do
     expect(HomeFeed.new(jeff).get(10)).to_not include(@status.id)
   end
 
-  it 'sends PuSH update to PuSH subscribers' do
-    expect(a_request(:post, 'http://example.com/push').with { |req|
-      req.body.match(OStatus::TagManager::VERBS[:delete])
-    }).to have_been_made
-  end
-
   it 'sends delete activity to followers' do
     expect(a_request(:post, 'http://example.com/inbox')).to have_been_made.twice
   end
 
-  it 'sends Salmon slap to previously mentioned users' do
-    expect(a_request(:post, "http://example.com/salmon").with { |req|
-      xml = OStatus2::Salmon.new.unpack(req.body)
-      xml.match(OStatus::TagManager::VERBS[:delete])
-    }).to have_been_made.once
-  end
-
   it 'sends delete activity to rebloggers' do
     expect(a_request(:post, 'http://example2.com/inbox')).to have_been_made
   end
diff --git a/spec/services/resolve_account_service_spec.rb b/spec/services/resolve_account_service_spec.rb
index 27a85af7c..7a64f4161 100644
--- a/spec/services/resolve_account_service_spec.rb
+++ b/spec/services/resolve_account_service_spec.rb
@@ -6,19 +6,13 @@ RSpec.describe ResolveAccountService, type: :service do
   before do
     stub_request(:get, "https://quitter.no/.well-known/host-meta").to_return(request_fixture('.host-meta.txt'))
     stub_request(:get, "https://example.com/.well-known/webfinger?resource=acct:catsrgr8@example.com").to_return(status: 404)
-    stub_request(:get, "https://redirected.com/.well-known/host-meta").to_return(request_fixture('redirected.host-meta.txt'))
     stub_request(:get, "https://example.com/.well-known/host-meta").to_return(status: 404)
-    stub_request(:get, "https://quitter.no/.well-known/webfinger?resource=acct:gargron@quitter.no").to_return(request_fixture('webfinger.txt'))
-    stub_request(:get, "https://redirected.com/.well-known/webfinger?resource=acct:gargron@redirected.com").to_return(request_fixture('webfinger.txt'))
-    stub_request(:get, "https://redirected.com/.well-known/webfinger?resource=acct:hacker1@redirected.com").to_return(request_fixture('webfinger-hacker1.txt'))
-    stub_request(:get, "https://redirected.com/.well-known/webfinger?resource=acct:hacker2@redirected.com").to_return(request_fixture('webfinger-hacker2.txt'))
-    stub_request(:get, "https://quitter.no/.well-known/webfinger?resource=acct:catsrgr8@quitter.no").to_return(status: 404)
-    stub_request(:get, "https://quitter.no/api/statuses/user_timeline/7477.atom").to_return(request_fixture('feed.txt'))
     stub_request(:get, "https://quitter.no/avatar/7477-300-20160211190340.png").to_return(request_fixture('avatar.txt'))
-    stub_request(:get, "https://localdomain.com/.well-known/host-meta").to_return(request_fixture('localdomain-hostmeta.txt'))
-    stub_request(:get, "https://localdomain.com/.well-known/webfinger?resource=acct:foo@localdomain.com").to_return(status: 404)
-    stub_request(:get, "https://webdomain.com/.well-known/webfinger?resource=acct:foo@localdomain.com").to_return(request_fixture('localdomain-webfinger.txt'))
-    stub_request(:get, "https://webdomain.com/users/foo.atom").to_return(request_fixture('localdomain-feed.txt'))
+    stub_request(:get, "https://quitter.no/.well-known/webfinger?resource=acct:catsrgr8@quitter.no").to_return(status: 404)
+    stub_request(:get, "https://ap.example.com/.well-known/webfinger?resource=acct:foo@ap.example.com").to_return(request_fixture('activitypub-webfinger.txt'))
+    stub_request(:get, "https://ap.example.com/users/foo").to_return(request_fixture('activitypub-actor.txt'))
+    stub_request(:get, "https://ap.example.com/users/foo.atom").to_return(request_fixture('activitypub-feed.txt'))
+    stub_request(:get, %r{https://ap.example.com/users/foo/\w+}).to_return(status: 404)
   end
 
   it 'raises error if no such user can be resolved via webfinger' do
@@ -29,74 +23,7 @@ RSpec.describe ResolveAccountService, type: :service do
     expect(subject.call('catsrgr8@example.com')).to be_nil
   end
 
-  it 'prevents hijacking existing accounts' do
-    account = subject.call('hacker1@redirected.com')
-    expect(account.salmon_url).to_not eq 'https://hacker.com/main/salmon/user/7477'
-  end
-
-  it 'prevents hijacking inexisting accounts' do
-    expect(subject.call('hacker2@redirected.com')).to be_nil
-  end
-
-  context 'with an OStatus account' do
-    it 'returns an already existing remote account' do
-      old_account      = Fabricate(:account, username: 'gargron', domain: 'quitter.no')
-      returned_account = subject.call('gargron@quitter.no')
-
-      expect(old_account.id).to eq returned_account.id
-    end
-
-    it 'returns a new remote account' do
-      account = subject.call('gargron@quitter.no')
-
-      expect(account.username).to eq 'gargron'
-      expect(account.domain).to eq 'quitter.no'
-      expect(account.remote_url).to eq 'https://quitter.no/api/statuses/user_timeline/7477.atom'
-    end
-
-    it 'follows a legitimate account redirection' do
-      account = subject.call('gargron@redirected.com')
-
-      expect(account.username).to eq 'gargron'
-      expect(account.domain).to eq 'quitter.no'
-      expect(account.remote_url).to eq 'https://quitter.no/api/statuses/user_timeline/7477.atom'
-    end
-
-    it 'returns a new remote account' do
-      account = subject.call('foo@localdomain.com')
-
-      expect(account.username).to eq 'foo'
-      expect(account.domain).to eq 'localdomain.com'
-      expect(account.remote_url).to eq 'https://webdomain.com/users/foo.atom'
-    end
-  end
-
   context 'with an ActivityPub account' do
-    before do
-      stub_request(:get, "https://ap.example.com/.well-known/webfinger?resource=acct:foo@ap.example.com").to_return(request_fixture('activitypub-webfinger.txt'))
-      stub_request(:get, "https://ap.example.com/users/foo").to_return(request_fixture('activitypub-actor.txt'))
-      stub_request(:get, "https://ap.example.com/users/foo.atom").to_return(request_fixture('activitypub-feed.txt'))
-      stub_request(:get, %r{https://ap.example.com/users/foo/\w+}).to_return(status: 404)
-    end
-
-    it 'fallback to OStatus if actor json could not be fetched' do
-      stub_request(:get, "https://ap.example.com/users/foo").to_return(status: 404)
-
-      account = subject.call('foo@ap.example.com')
-
-      expect(account.ostatus?).to eq true
-      expect(account.remote_url).to eq 'https://ap.example.com/users/foo.atom'
-    end
-
-    it 'fallback to OStatus if actor json did not have inbox_url' do
-      stub_request(:get, "https://ap.example.com/users/foo").to_return(request_fixture('activitypub-actor-noinbox.txt'))
-
-      account = subject.call('foo@ap.example.com')
-
-      expect(account.ostatus?).to eq true
-      expect(account.remote_url).to eq 'https://ap.example.com/users/foo.atom'
-    end
-
     it 'returns new remote account' do
       account = subject.call('foo@ap.example.com')
 
@@ -124,13 +51,14 @@ RSpec.describe ResolveAccountService, type: :service do
   it 'processes one remote account at a time using locks' do
     wait_for_start = true
     fail_occurred  = false
-    return_values  = []
+    return_values  = Concurrent::Array.new
 
     threads = Array.new(5) do
       Thread.new do
         true while wait_for_start
+
         begin
-          return_values << described_class.new.call('foo@localdomain.com')
+          return_values << described_class.new.call('foo@ap.example.com')
         rescue ActiveRecord::RecordNotUnique
           fail_occurred = true
         end
diff --git a/spec/services/send_interaction_service_spec.rb b/spec/services/send_interaction_service_spec.rb
deleted file mode 100644
index 710d8184c..000000000
--- a/spec/services/send_interaction_service_spec.rb
+++ /dev/null
@@ -1,7 +0,0 @@
-require 'rails_helper'
-
-RSpec.describe SendInteractionService, type: :service do
-  subject { SendInteractionService.new }
-
-  it 'sends an XML envelope to the Salmon end point of remote user'
-end
diff --git a/spec/services/subscribe_service_spec.rb b/spec/services/subscribe_service_spec.rb
deleted file mode 100644
index 10bdb1ba8..000000000
--- a/spec/services/subscribe_service_spec.rb
+++ /dev/null
@@ -1,43 +0,0 @@
-require 'rails_helper'
-
-RSpec.describe SubscribeService, type: :service do
-  let(:account) { Fabricate(:account, username: 'bob', domain: 'example.com', hub_url: 'http://hub.example.com') }
-  subject { SubscribeService.new }
-
-  it 'sends subscription request to PuSH hub' do
-    stub_request(:post, 'http://hub.example.com/').to_return(status: 202)
-    subject.call(account)
-    expect(a_request(:post, 'http://hub.example.com/')).to have_been_made.once
-  end
-
-  it 'generates and keeps PuSH secret on successful call' do
-    stub_request(:post, 'http://hub.example.com/').to_return(status: 202)
-    subject.call(account)
-    expect(account.secret).to_not be_blank
-  end
-
-  it 'fails silently if PuSH hub forbids subscription' do
-    stub_request(:post, 'http://hub.example.com/').to_return(status: 403)
-    subject.call(account)
-  end
-
-  it 'fails silently if PuSH hub is not found' do
-    stub_request(:post, 'http://hub.example.com/').to_return(status: 404)
-    subject.call(account)
-  end
-
-  it 'fails loudly if there is a network error' do
-    stub_request(:post, 'http://hub.example.com/').to_raise(HTTP::Error)
-    expect { subject.call(account) }.to raise_error HTTP::Error
-  end
-
-  it 'fails loudly if PuSH hub is unavailable' do
-    stub_request(:post, 'http://hub.example.com/').to_return(status: 503)
-    expect { subject.call(account) }.to raise_error Mastodon::UnexpectedResponseError
-  end
-
-  it 'fails loudly if rate limited' do
-    stub_request(:post, 'http://hub.example.com/').to_return(status: 429)
-    expect { subject.call(account) }.to raise_error Mastodon::UnexpectedResponseError
-  end
-end
diff --git a/spec/services/unblock_service_spec.rb b/spec/services/unblock_service_spec.rb
index 5835b912b..6350c6834 100644
--- a/spec/services/unblock_service_spec.rb
+++ b/spec/services/unblock_service_spec.rb
@@ -30,13 +30,6 @@ RSpec.describe UnblockService, type: :service do
     it 'destroys the blocking relation' do
       expect(sender.blocking?(bob)).to be false
     end
-
-    it 'sends an unblock salmon slap' do
-      expect(a_request(:post, "http://salmon.example.com/").with { |req|
-        xml = OStatus2::Salmon.new.unpack(req.body)
-        xml.match(OStatus::TagManager::VERBS[:unblock])
-      }).to have_been_made.once
-    end
   end
 
   describe 'remote ActivityPub' do
diff --git a/spec/services/unfollow_service_spec.rb b/spec/services/unfollow_service_spec.rb
index 8a2881ab1..84b5dafbc 100644
--- a/spec/services/unfollow_service_spec.rb
+++ b/spec/services/unfollow_service_spec.rb
@@ -30,13 +30,6 @@ RSpec.describe UnfollowService, type: :service do
     it 'destroys the following relation' do
       expect(sender.following?(bob)).to be false
     end
-
-    it 'sends an unfollow salmon slap' do
-      expect(a_request(:post, "http://salmon.example.com/").with { |req|
-        xml = OStatus2::Salmon.new.unpack(req.body)
-        xml.match(OStatus::TagManager::VERBS[:unfollow])
-      }).to have_been_made.once
-    end
   end
 
   describe 'remote ActivityPub' do
diff --git a/spec/services/unsubscribe_service_spec.rb b/spec/services/unsubscribe_service_spec.rb
deleted file mode 100644
index 54d4b1b53..000000000
--- a/spec/services/unsubscribe_service_spec.rb
+++ /dev/null
@@ -1,37 +0,0 @@
-require 'rails_helper'
-
-RSpec.describe UnsubscribeService, type: :service do
-  let(:account) { Fabricate(:account, username: 'bob', domain: 'example.com', hub_url: 'http://hub.example.com') }
-  subject { UnsubscribeService.new }
-
-  it 'removes the secret and resets expiration on account' do
-    stub_request(:post, 'http://hub.example.com/').to_return(status: 204)
-    subject.call(account)
-    account.reload
-
-    expect(account.secret).to be_blank
-    expect(account.subscription_expires_at).to be_blank
-  end
-
-  it 'logs error on subscription failure' do
-    logger = stub_logger
-    stub_request(:post, 'http://hub.example.com/').to_return(status: 404)
-    subject.call(account)
-
-    expect(logger).to have_received(:debug).with(/unsubscribe for bob@example.com failed/)
-  end
-
-  it 'logs error on connection failure' do
-    logger = stub_logger
-    stub_request(:post, 'http://hub.example.com/').to_raise(HTTP::Error)
-    subject.call(account)
-
-    expect(logger).to have_received(:debug).with(/unsubscribe for bob@example.com failed/)
-  end
-
-  def stub_logger
-    double(debug: nil).tap do |logger|
-      allow(Rails).to receive(:logger).and_return(logger)
-    end
-  end
-end
diff --git a/spec/services/update_remote_profile_service_spec.rb b/spec/services/update_remote_profile_service_spec.rb
deleted file mode 100644
index f3ea70b80..000000000
--- a/spec/services/update_remote_profile_service_spec.rb
+++ /dev/null
@@ -1,84 +0,0 @@
-require 'rails_helper'
-
-RSpec.describe UpdateRemoteProfileService, type: :service do
-  let(:xml) { File.read(Rails.root.join('spec', 'fixtures', 'push', 'feed.atom')) }
-
-  subject { UpdateRemoteProfileService.new }
-
-  before do
-    stub_request(:get, 'https://quitter.no/avatar/7477-300-20160211190340.png').to_return(request_fixture('avatar.txt'))
-  end
-
-  context 'with updated details' do
-    let(:remote_account) { Fabricate(:account, username: 'bob', domain: 'example.com') }
-
-    before do
-      subject.call(xml, remote_account)
-    end
-
-    it 'downloads new avatar' do
-      expect(a_request(:get, 'https://quitter.no/avatar/7477-300-20160211190340.png')).to have_been_made
-    end
-
-    it 'sets the avatar remote url' do
-      expect(remote_account.reload.avatar_remote_url).to eq 'https://quitter.no/avatar/7477-300-20160211190340.png'
-    end
-
-    it 'sets display name' do
-      expect(remote_account.reload.display_name).to eq 'ＤＩＧＩＴＡＬ ＣＡＴ'
-    end
-
-    it 'sets note' do
-      expect(remote_account.reload.note).to eq 'Software engineer, free time musician and ＤＩＧＩＴＡＬ ＳＰＯＲＴＳ enthusiast. Likes cats. Warning: May contain memes'
-    end
-  end
-
-  context 'with unchanged details' do
-    let(:remote_account) { Fabricate(:account, username: 'bob', domain: 'example.com', display_name: 'ＤＩＧＩＴＡＬ ＣＡＴ', note: 'Software engineer, free time musician and ＤＩＧＩＴＡＬ ＳＰＯＲＴＳ enthusiast. Likes cats. Warning: May contain memes', avatar_remote_url: 'https://quitter.no/avatar/7477-300-20160211190340.png') }
-
-    before do
-      subject.call(xml, remote_account)
-    end
-
-    it 'does not re-download avatar' do
-      expect(a_request(:get, 'https://quitter.no/avatar/7477-300-20160211190340.png')).to have_been_made.once
-    end
-
-    it 'sets the avatar remote url' do
-      expect(remote_account.reload.avatar_remote_url).to eq 'https://quitter.no/avatar/7477-300-20160211190340.png'
-    end
-
-    it 'sets display name' do
-      expect(remote_account.reload.display_name).to eq 'ＤＩＧＩＴＡＬ ＣＡＴ'
-    end
-
-    it 'sets note' do
-      expect(remote_account.reload.note).to eq 'Software engineer, free time musician and ＤＩＧＩＴＡＬ ＳＰＯＲＴＳ enthusiast. Likes cats. Warning: May contain memes'
-    end
-  end
-
-  context 'with updated details from a domain set to reject media' do
-    let(:remote_account) { Fabricate(:account, username: 'bob', domain: 'example.com') }
-    let!(:domain_block) { Fabricate(:domain_block, domain: 'example.com', reject_media: true) }
-
-    before do
-      subject.call(xml, remote_account)
-    end
-
-    it 'does not the avatar remote url' do
-      expect(remote_account.reload.avatar_remote_url).to be_nil
-    end
-
-    it 'sets display name' do
-      expect(remote_account.reload.display_name).to eq 'ＤＩＧＩＴＡＬ ＣＡＴ'
-    end
-
-    it 'sets note' do
-      expect(remote_account.reload.note).to eq 'Software engineer, free time musician and ＤＩＧＩＴＡＬ ＳＰＯＲＴＳ enthusiast. Likes cats. Warning: May contain memes'
-    end
-
-    it 'does not set store the avatar' do
-      expect(remote_account.reload.avatar_file_name).to be_nil
-    end
-  end
-end
diff --git a/spec/workers/after_remote_follow_request_worker_spec.rb b/spec/workers/after_remote_follow_request_worker_spec.rb
deleted file mode 100644
index bd623cca5..000000000
--- a/spec/workers/after_remote_follow_request_worker_spec.rb
+++ /dev/null
@@ -1,59 +0,0 @@
-# frozen_string_literal: true
-
-require 'rails_helper'
-
-describe AfterRemoteFollowRequestWorker do
-  subject { described_class.new }
-  let(:follow_request) { Fabricate(:follow_request) }
-  describe 'perform' do
-    context 'when the follow_request does not exist' do
-      it 'catches a raise and returns true' do
-        allow(FollowService).to receive(:new)
-        result = subject.perform('aaa')
-
-        expect(result).to eq(true)
-        expect(FollowService).not_to have_received(:new)
-      end
-    end
-
-    context 'when the account cannot be updated' do
-      it 'returns nil and does not call service when account is nil' do
-        allow(FollowService).to receive(:new)
-        service = double(call: nil)
-        allow(FetchRemoteAccountService).to receive(:new).and_return(service)
-
-        result = subject.perform(follow_request.id)
-
-        expect(result).to be_nil
-        expect(FollowService).not_to have_received(:new)
-      end
-
-      it 'returns nil and does not call service when account is locked' do
-        allow(FollowService).to receive(:new)
-        service = double(call: double(locked?: true))
-        allow(FetchRemoteAccountService).to receive(:new).and_return(service)
-
-        result = subject.perform(follow_request.id)
-
-        expect(result).to be_nil
-        expect(FollowService).not_to have_received(:new)
-      end
-    end
-
-    context 'when the account is updated' do
-      it 'calls the follow service and destroys the follow' do
-        follow_service = double(call: nil)
-        allow(FollowService).to receive(:new).and_return(follow_service)
-        account = Fabricate(:account, locked: false)
-        service = double(call: account)
-        allow(FetchRemoteAccountService).to receive(:new).and_return(service)
-
-        result = subject.perform(follow_request.id)
-
-        expect(result).to be_nil
-        expect(follow_service).to have_received(:call).with(follow_request.account, account.acct)
-        expect { follow_request.reload }.to raise_error(ActiveRecord::RecordNotFound)
-      end
-    end
-  end
-end
diff --git a/spec/workers/after_remote_follow_worker_spec.rb b/spec/workers/after_remote_follow_worker_spec.rb
deleted file mode 100644
index d93c469f9..000000000
--- a/spec/workers/after_remote_follow_worker_spec.rb
+++ /dev/null
@@ -1,59 +0,0 @@
-# frozen_string_literal: true
-
-require 'rails_helper'
-
-describe AfterRemoteFollowWorker do
-  subject { described_class.new }
-  let(:follow) { Fabricate(:follow) }
-  describe 'perform' do
-    context 'when the follow does not exist' do
-      it 'catches a raise and returns true' do
-        allow(FollowService).to receive(:new)
-        result = subject.perform('aaa')
-
-        expect(result).to eq(true)
-        expect(FollowService).not_to have_received(:new)
-      end
-    end
-
-    context 'when the account cannot be updated' do
-      it 'returns nil and does not call service when account is nil' do
-        allow(FollowService).to receive(:new)
-        service = double(call: nil)
-        allow(FetchRemoteAccountService).to receive(:new).and_return(service)
-
-        result = subject.perform(follow.id)
-
-        expect(result).to be_nil
-        expect(FollowService).not_to have_received(:new)
-      end
-
-      it 'returns nil and does not call service when account is not locked' do
-        allow(FollowService).to receive(:new)
-        service = double(call: double(locked?: false))
-        allow(FetchRemoteAccountService).to receive(:new).and_return(service)
-
-        result = subject.perform(follow.id)
-
-        expect(result).to be_nil
-        expect(FollowService).not_to have_received(:new)
-      end
-    end
-
-    context 'when the account is updated' do
-      it 'calls the follow service and destroys the follow' do
-        follow_service = double(call: nil)
-        allow(FollowService).to receive(:new).and_return(follow_service)
-        account = Fabricate(:account, locked: true)
-        service = double(call: account)
-        allow(FetchRemoteAccountService).to receive(:new).and_return(service)
-
-        result = subject.perform(follow.id)
-
-        expect(result).to be_nil
-        expect(follow_service).to have_received(:call).with(follow.account, account.acct)
-        expect { follow.reload }.to raise_error(ActiveRecord::RecordNotFound)
-      end
-    end
-  end
-end
diff --git a/spec/workers/pubsubhubbub/confirmation_worker_spec.rb b/spec/workers/pubsubhubbub/confirmation_worker_spec.rb
deleted file mode 100644
index 1eecdd2b5..000000000
--- a/spec/workers/pubsubhubbub/confirmation_worker_spec.rb
+++ /dev/null
@@ -1,88 +0,0 @@
-# frozen_string_literal: true
-
-require 'rails_helper'
-
-describe Pubsubhubbub::ConfirmationWorker do
-  include RoutingHelper
-
-  subject { described_class.new }
-
-  let!(:alice) { Fabricate(:account, username: 'alice') }
-  let!(:subscription) { Fabricate(:subscription, account: alice, callback_url: 'http://example.com/api', confirmed: false, expires_at: 3.days.from_now, secret: nil) }
-
-  describe 'perform' do
-    describe 'with subscribe mode' do
-      it 'confirms and updates subscription when challenge matches' do
-        stub_random_value
-        stub_request(:get, url_for_mode('subscribe'))
-          .with(headers: http_headers)
-          .to_return(status: 200, body: challenge_value, headers: {})
-
-        seconds = 10.days.seconds.to_i
-        subject.perform(subscription.id, 'subscribe', 'asdf', seconds)
-
-        subscription.reload
-        expect(subscription.secret).to eq 'asdf'
-        expect(subscription.confirmed).to eq true
-        expect(subscription.expires_at).to be_within(5).of(10.days.from_now)
-      end
-
-      it 'does not update subscription when challenge does not match' do
-        stub_random_value
-        stub_request(:get, url_for_mode('subscribe'))
-          .with(headers: http_headers)
-          .to_return(status: 200, body: 'wrong value', headers: {})
-
-        seconds = 10.days.seconds.to_i
-        subject.perform(subscription.id, 'subscribe', 'asdf', seconds)
-
-        subscription.reload
-        expect(subscription.secret).to be_blank
-        expect(subscription.confirmed).to eq false
-        expect(subscription.expires_at).to be_within(5).of(3.days.from_now)
-      end
-    end
-
-    describe 'with unsubscribe mode' do
-      it 'confirms and destroys subscription when challenge matches' do
-        stub_random_value
-        stub_request(:get, url_for_mode('unsubscribe'))
-          .with(headers: http_headers)
-          .to_return(status: 200, body: challenge_value, headers: {})
-
-        seconds = 10.days.seconds.to_i
-        subject.perform(subscription.id, 'unsubscribe', 'asdf', seconds)
-
-        expect { subscription.reload }.to raise_error(ActiveRecord::RecordNotFound)
-      end
-
-      it 'does not destroy subscription when challenge does not match' do
-        stub_random_value
-        stub_request(:get, url_for_mode('unsubscribe'))
-          .with(headers: http_headers)
-          .to_return(status: 200, body: 'wrong value', headers: {})
-
-        seconds = 10.days.seconds.to_i
-        subject.perform(subscription.id, 'unsubscribe', 'asdf', seconds)
-
-        expect { subscription.reload }.not_to raise_error
-      end
-    end
-  end
-
-  def url_for_mode(mode)
-    "http://example.com/api?hub.challenge=#{challenge_value}&hub.lease_seconds=863999&hub.mode=#{mode}&hub.topic=https://#{Rails.configuration.x.local_domain}/users/alice.atom"
-  end
-
-  def stub_random_value
-    allow(SecureRandom).to receive(:hex).and_return(challenge_value)
-  end
-
-  def challenge_value
-    '1a2s3d4f'
-  end
-
-  def http_headers
-    { 'Connection' => 'close', 'Host' => 'example.com' }
-  end
-end
diff --git a/spec/workers/pubsubhubbub/delivery_worker_spec.rb b/spec/workers/pubsubhubbub/delivery_worker_spec.rb
deleted file mode 100644
index c0e0d5186..000000000
--- a/spec/workers/pubsubhubbub/delivery_worker_spec.rb
+++ /dev/null
@@ -1,68 +0,0 @@
-# frozen_string_literal: true
-
-require 'rails_helper'
-
-describe Pubsubhubbub::DeliveryWorker do
-  include RoutingHelper
-  subject { described_class.new }
-
-  let(:payload) { 'test' }
-
-  describe 'perform' do
-    it 'raises when subscription does not exist' do
-      expect { subject.perform 123, payload }.to raise_error(ActiveRecord::RecordNotFound)
-    end
-
-    it 'does not attempt to deliver when domain blocked' do
-      _domain_block = Fabricate(:domain_block, domain: 'example.com', severity: :suspend)
-      subscription = Fabricate(:subscription, callback_url: 'https://example.com/api', last_successful_delivery_at: 2.days.ago)
-
-      subject.perform(subscription.id, payload)
-
-      expect(subscription.reload.last_successful_delivery_at).to be_within(2).of(2.days.ago)
-    end
-
-    it 'raises when request fails' do
-      subscription = Fabricate(:subscription)
-
-      stub_request_to_respond_with(subscription, 500)
-      expect { subject.perform(subscription.id, payload) }.to raise_error Mastodon::UnexpectedResponseError
-    end
-
-    it 'updates subscriptions when delivery succeeds' do
-      subscription = Fabricate(:subscription)
-
-      stub_request_to_respond_with(subscription, 200)
-      subject.perform(subscription.id, payload)
-
-      expect(subscription.reload.last_successful_delivery_at).to be_within(2).of(Time.now.utc)
-    end
-
-    it 'updates subscription without a secret when delivery succeeds' do
-      subscription = Fabricate(:subscription, secret: nil)
-
-      stub_request_to_respond_with(subscription, 200)
-      subject.perform(subscription.id, payload)
-
-      expect(subscription.reload.last_successful_delivery_at).to be_within(2).of(Time.now.utc)
-    end
-
-    def stub_request_to_respond_with(subscription, code)
-      stub_request(:post, 'http://example.com/callback')
-        .with(body: payload, headers: expected_headers(subscription))
-        .to_return(status: code, body: '', headers: {})
-    end
-
-    def expected_headers(subscription)
-      {
-        'Connection' => 'close',
-        'Content-Type' => 'application/atom+xml',
-        'Host' => 'example.com',
-        'Link' => "<https://#{Rails.configuration.x.local_domain}/api/push>; rel=\"hub\", <https://#{Rails.configuration.x.local_domain}/users/#{subscription.account.username}.atom>; rel=\"self\"",
-      }.tap do |basic|
-        known_digest = OpenSSL::HMAC.hexdigest(OpenSSL::Digest.new('sha1'), subscription.secret.to_s, payload)
-        basic.merge('X-Hub-Signature' => "sha1=#{known_digest}") if subscription.secret?
-      end
-    end
-  end
-end
diff --git a/spec/workers/pubsubhubbub/distribution_worker_spec.rb b/spec/workers/pubsubhubbub/distribution_worker_spec.rb
deleted file mode 100644
index 584485079..000000000
--- a/spec/workers/pubsubhubbub/distribution_worker_spec.rb
+++ /dev/null
@@ -1,46 +0,0 @@
-require 'rails_helper'
-
-describe Pubsubhubbub::DistributionWorker do
-  subject { Pubsubhubbub::DistributionWorker.new }
-
-  let!(:alice) { Fabricate(:account, username: 'alice') }
-  let!(:bob) { Fabricate(:account, username: 'bob', domain: 'example2.com') }
-  let!(:anonymous_subscription) { Fabricate(:subscription, account: alice, callback_url: 'http://example1.com', confirmed: true, lease_seconds: 3600) }
-  let!(:subscription_with_follower) { Fabricate(:subscription, account: alice, callback_url: 'http://example2.com', confirmed: true, lease_seconds: 3600) }
-
-  before do
-    bob.follow!(alice)
-  end
-
-  describe 'with public status' do
-    let(:status) { Fabricate(:status, account: alice, text: 'Hello', visibility: :public) }
-
-    it 'delivers payload to all subscriptions' do
-      allow(Pubsubhubbub::DeliveryWorker).to receive(:push_bulk)
-      subject.perform(status.stream_entry.id)
-      expect(Pubsubhubbub::DeliveryWorker).to have_received(:push_bulk).with([anonymous_subscription.id, subscription_with_follower.id])
-    end
-  end
-
-  context 'when OStatus privacy is not used' do
-    describe 'with private status' do
-      let(:status) { Fabricate(:status, account: alice, text: 'Hello', visibility: :private) }
-
-      it 'does not deliver anything' do
-        allow(Pubsubhubbub::DeliveryWorker).to receive(:push_bulk)
-        subject.perform(status.stream_entry.id)
-        expect(Pubsubhubbub::DeliveryWorker).to_not have_received(:push_bulk)
-      end
-    end
-
-    describe 'with direct status' do
-      let(:status) { Fabricate(:status, account: alice, text: 'Hello', visibility: :direct) }
-
-      it 'does not deliver payload' do
-        allow(Pubsubhubbub::DeliveryWorker).to receive(:push_bulk)
-        subject.perform(status.stream_entry.id)
-        expect(Pubsubhubbub::DeliveryWorker).to_not have_received(:push_bulk)
-      end
-    end
-  end
-end
diff --git a/spec/workers/scheduler/subscriptions_scheduler_spec.rb b/spec/workers/scheduler/subscriptions_scheduler_spec.rb
deleted file mode 100644
index a7d1046de..000000000
--- a/spec/workers/scheduler/subscriptions_scheduler_spec.rb
+++ /dev/null
@@ -1,19 +0,0 @@
-require 'rails_helper'
-
-describe Scheduler::SubscriptionsScheduler do
-  subject { Scheduler::SubscriptionsScheduler.new }
-
-  let!(:expiring_account1) { Fabricate(:account, subscription_expires_at: 20.minutes.from_now, domain: 'example.com', followers_count: 1, hub_url: 'http://hub.example.com') }
-  let!(:expiring_account2) { Fabricate(:account, subscription_expires_at: 4.hours.from_now, domain: 'example.org', followers_count: 1, hub_url: 'http://hub.example.org') }
-
-  before do
-    stub_request(:post, 'http://hub.example.com/').to_return(status: 202)
-    stub_request(:post, 'http://hub.example.org/').to_return(status: 202)
-  end
-
-  it 're-subscribes for all expiring accounts' do
-    subject.perform
-    expect(a_request(:post, 'http://hub.example.com/')).to have_been_made.once
-    expect(a_request(:post, 'http://hub.example.org/')).to have_been_made.once
-  end
-end
-- 
cgit 


From b8514561394767a10d3cf40132ada24d938c1680 Mon Sep 17 00:00:00 2001
From: Eugen Rochko <eugen@zeonfederated.com>
Date: Sun, 7 Jul 2019 16:16:51 +0200
Subject: Remove Atom feeds and old URLs in the form of `GET
 /:username/updates/:id` (#11247)

---
 app/controllers/accounts_controller.rb             |    7 -
 app/controllers/statuses_controller.rb             |   36 +-
 app/controllers/stream_entries_controller.rb       |   64 -
 app/helpers/admin/action_logs_helper.rb            |    2 +-
 app/helpers/home_helper.rb                         |    2 +-
 app/helpers/statuses_helper.rb                     |  222 +++
 app/helpers/stream_entries_helper.rb               |  220 ---
 app/javascript/styles/application.scss             |    2 +-
 app/javascript/styles/mastodon/statuses.scss       |  163 +++
 app/javascript/styles/mastodon/stream_entries.scss |  163 ---
 app/lib/formatter.rb                               |    2 +-
 app/lib/ostatus/atom_serializer.rb                 |  376 ------
 app/lib/status_finder.rb                           |    2 -
 app/lib/tag_manager.rb                             |   11 -
 app/mailers/admin_mailer.rb                        |    2 +-
 app/mailers/notification_mailer.rb                 |    2 +-
 app/models/concerns/account_associations.rb        |    1 -
 app/models/concerns/streamable.rb                  |   43 -
 app/models/remote_profile.rb                       |   57 -
 app/models/status.rb                               |    4 -
 app/models/stream_entry.rb                         |   59 -
 app/serializers/rest/account_serializer.rb         |    2 +-
 app/serializers/rest/status_serializer.rb          |    6 +-
 app/serializers/rss/account_serializer.rb          |    6 +-
 app/serializers/rss/tag_serializer.rb              |    4 +-
 app/services/batched_remove_status_service.rb      |    2 +-
 app/services/fetch_link_card_service.rb            |    2 +-
 app/services/process_mentions_service.rb           |    2 +-
 app/services/remove_status_service.rb              |   15 +-
 app/services/resolve_url_service.rb                |    5 +-
 app/services/suspend_account_service.rb            |    1 -
 app/views/accounts/_moved.html.haml                |    4 +-
 app/views/accounts/show.html.haml                  |    4 +-
 app/views/admin/accounts/_account.html.haml        |    2 +-
 app/views/admin/reports/_status.html.haml          |    2 +-
 app/views/application/_card.html.haml              |    2 +-
 .../_post_follow_actions.html.haml                 |    2 +-
 app/views/remote_interaction/new.html.haml         |    2 +-
 app/views/remote_unfollows/_card.html.haml         |    2 +-
 .../_post_follow_actions.html.haml                 |    2 +-
 app/views/statuses/_attachment_list.html.haml      |    8 +
 app/views/statuses/_detailed_status.html.haml      |   79 ++
 app/views/statuses/_og_description.html.haml       |    4 +
 app/views/statuses/_og_image.html.haml             |   38 +
 app/views/statuses/_poll.html.haml                 |   27 +
 app/views/statuses/_simple_status.html.haml        |   60 +
 app/views/statuses/_status.html.haml               |   62 +
 app/views/statuses/embed.html.haml                 |    3 +
 app/views/statuses/show.html.haml                  |   24 +
 .../stream_entries/_attachment_list.html.haml      |    8 -
 .../stream_entries/_detailed_status.html.haml      |   79 --
 app/views/stream_entries/_og_description.html.haml |    4 -
 app/views/stream_entries/_og_image.html.haml       |   38 -
 app/views/stream_entries/_poll.html.haml           |   27 -
 app/views/stream_entries/_simple_status.html.haml  |   60 -
 app/views/stream_entries/_status.html.haml         |   62 -
 app/views/stream_entries/embed.html.haml           |    3 -
 app/views/stream_entries/show.html.haml            |   25 -
 config/routes.rb                                   |    6 -
 .../20190706233204_drop_stream_entries.rb          |   13 +
 db/schema.rb                                       |   14 +-
 spec/controllers/accounts_controller_spec.rb       |   31 -
 spec/controllers/api/oembed_controller_spec.rb     |    2 +-
 spec/controllers/application_controller_spec.rb    |    4 -
 spec/controllers/statuses_controller_spec.rb       |   16 +-
 spec/controllers/stream_entries_controller_spec.rb |   95 --
 spec/fabricators/stream_entry_fabricator.rb        |    5 -
 .../admin/account_moderation_notes_helper_spec.rb  |    2 +-
 spec/helpers/statuses_helper_spec.rb               |  224 ++++
 spec/helpers/stream_entries_helper_spec.rb         |  224 ----
 spec/lib/activitypub/tag_manager_spec.rb           |    6 -
 spec/lib/ostatus/atom_serializer_spec.rb           | 1415 --------------------
 spec/lib/status_finder_spec.rb                     |    9 -
 spec/lib/tag_manager_spec.rb                       |   42 -
 spec/models/concerns/streamable_spec.rb            |   63 -
 spec/models/remote_profile_spec.rb                 |  143 --
 spec/models/stream_entry_spec.rb                   |  192 ---
 spec/services/process_mentions_service_spec.rb     |    4 +-
 spec/services/suspend_account_service_spec.rb      |    6 +-
 spec/views/statuses/show.html.haml_spec.rb         |   81 ++
 spec/views/stream_entries/show.html.haml_spec.rb   |   88 --
 81 files changed, 1071 insertions(+), 3732 deletions(-)
 delete mode 100644 app/controllers/stream_entries_controller.rb
 create mode 100644 app/helpers/statuses_helper.rb
 delete mode 100644 app/helpers/stream_entries_helper.rb
 create mode 100644 app/javascript/styles/mastodon/statuses.scss
 delete mode 100644 app/javascript/styles/mastodon/stream_entries.scss
 delete mode 100644 app/lib/ostatus/atom_serializer.rb
 delete mode 100644 app/models/concerns/streamable.rb
 delete mode 100644 app/models/remote_profile.rb
 delete mode 100644 app/models/stream_entry.rb
 create mode 100644 app/views/statuses/_attachment_list.html.haml
 create mode 100644 app/views/statuses/_detailed_status.html.haml
 create mode 100644 app/views/statuses/_og_description.html.haml
 create mode 100644 app/views/statuses/_og_image.html.haml
 create mode 100644 app/views/statuses/_poll.html.haml
 create mode 100644 app/views/statuses/_simple_status.html.haml
 create mode 100644 app/views/statuses/_status.html.haml
 create mode 100644 app/views/statuses/embed.html.haml
 create mode 100644 app/views/statuses/show.html.haml
 delete mode 100644 app/views/stream_entries/_attachment_list.html.haml
 delete mode 100644 app/views/stream_entries/_detailed_status.html.haml
 delete mode 100644 app/views/stream_entries/_og_description.html.haml
 delete mode 100644 app/views/stream_entries/_og_image.html.haml
 delete mode 100644 app/views/stream_entries/_poll.html.haml
 delete mode 100644 app/views/stream_entries/_simple_status.html.haml
 delete mode 100644 app/views/stream_entries/_status.html.haml
 delete mode 100644 app/views/stream_entries/embed.html.haml
 delete mode 100644 app/views/stream_entries/show.html.haml
 create mode 100644 db/post_migrate/20190706233204_drop_stream_entries.rb
 delete mode 100644 spec/controllers/stream_entries_controller_spec.rb
 delete mode 100644 spec/fabricators/stream_entry_fabricator.rb
 create mode 100644 spec/helpers/statuses_helper_spec.rb
 delete mode 100644 spec/helpers/stream_entries_helper_spec.rb
 delete mode 100644 spec/lib/ostatus/atom_serializer_spec.rb
 delete mode 100644 spec/models/concerns/streamable_spec.rb
 delete mode 100644 spec/models/remote_profile_spec.rb
 delete mode 100644 spec/models/stream_entry_spec.rb
 create mode 100644 spec/views/statuses/show.html.haml_spec.rb
 delete mode 100644 spec/views/stream_entries/show.html.haml_spec.rb

(limited to 'app/models')

diff --git a/app/controllers/accounts_controller.rb b/app/controllers/accounts_controller.rb
index 73a4b1859..065707378 100644
--- a/app/controllers/accounts_controller.rb
+++ b/app/controllers/accounts_controller.rb
@@ -31,13 +31,6 @@ class AccountsController < ApplicationController
         end
       end
 
-      format.atom do
-        mark_cacheable!
-
-        @entries = @account.stream_entries.where(hidden: false).with_includes.paginate_by_max_id(PAGE_SIZE, params[:max_id], params[:since_id])
-        render xml: OStatus::AtomSerializer.render(OStatus::AtomSerializer.new.feed(@account, @entries.reject { |entry| entry.status.nil? }))
-      end
-
       format.rss do
         mark_cacheable!
 
diff --git a/app/controllers/statuses_controller.rb b/app/controllers/statuses_controller.rb
index ef26691b2..776099ca8 100644
--- a/app/controllers/statuses_controller.rb
+++ b/app/controllers/statuses_controller.rb
@@ -33,12 +33,10 @@ class StatusesController < ApplicationController
 
         set_ancestors
         set_descendants
-
-        render 'stream_entries/show'
       end
 
       format.json do
-        render_cached_json(['activitypub', 'note', @status], content_type: 'application/activity+json', public: !@stream_entry.hidden?) do
+        render_cached_json(['activitypub', 'note', @status], content_type: 'application/activity+json', public: @status.distributable?) do
           ActiveModelSerializers::SerializableResource.new(@status, serializer: ActivityPub::NoteSerializer, adapter: ActivityPub::Adapter)
         end
       end
@@ -46,7 +44,7 @@ class StatusesController < ApplicationController
   end
 
   def activity
-    render_cached_json(['activitypub', 'activity', @status], content_type: 'application/activity+json', public: !@stream_entry.hidden?) do
+    render_cached_json(['activitypub', 'activity', @status], content_type: 'application/activity+json', public: @status.distributable?) do
       ActiveModelSerializers::SerializableResource.new(@status, serializer: ActivityPub::ActivitySerializer, adapter: ActivityPub::Adapter)
     end
   end
@@ -58,7 +56,7 @@ class StatusesController < ApplicationController
     response.headers['X-Frame-Options'] = 'ALLOWALL'
     @autoplay = ActiveModel::Type::Boolean.new.cast(params[:autoplay])
 
-    render 'stream_entries/embed', layout: 'embedded'
+    render layout: 'embedded'
   end
 
   def replies
@@ -92,11 +90,20 @@ class StatusesController < ApplicationController
 
   def create_descendant_thread(starting_depth, statuses)
     depth = starting_depth + statuses.size
+
     if depth < DESCENDANTS_DEPTH_LIMIT
-      { statuses: statuses, starting_depth: starting_depth }
+      {
+        statuses: statuses,
+        starting_depth: starting_depth,
+      }
     else
       next_status = statuses.pop
-      { statuses: statuses, starting_depth: starting_depth, next_status: next_status }
+
+      {
+        statuses: statuses,
+        starting_depth: starting_depth,
+        next_status: next_status,
+      }
     end
   end
 
@@ -164,22 +171,13 @@ class StatusesController < ApplicationController
   end
 
   def set_link_headers
-    response.headers['Link'] = LinkHeader.new(
-      [
-        [account_stream_entry_url(@account, @status.stream_entry, format: 'atom'), [%w(rel alternate), %w(type application/atom+xml)]],
-        [ActivityPub::TagManager.instance.uri_for(@status), [%w(rel alternate), %w(type application/activity+json)]],
-      ]
-    )
+    response.headers['Link'] = LinkHeader.new([[ActivityPub::TagManager.instance.uri_for(@status), [%w(rel alternate), %w(type application/activity+json)]]])
   end
 
   def set_status
-    @status       = @account.statuses.find(params[:id])
-    @stream_entry = @status.stream_entry
-    @type         = @stream_entry.activity_type.downcase
-
+    @status = @account.statuses.find(params[:id])
     authorize @status, :show?
   rescue Mastodon::NotPermittedError
-    # Reraise in order to get a 404
     raise ActiveRecord::RecordNotFound
   end
 
@@ -192,7 +190,7 @@ class StatusesController < ApplicationController
   end
 
   def redirect_to_original
-    redirect_to ::TagManager.instance.url_for(@status.reblog) if @status.reblog?
+    redirect_to ActivityPub::TagManager.instance.url_for(@status.reblog) if @status.reblog?
   end
 
   def set_referrer_policy_header
diff --git a/app/controllers/stream_entries_controller.rb b/app/controllers/stream_entries_controller.rb
deleted file mode 100644
index 0f7e9e0f5..000000000
--- a/app/controllers/stream_entries_controller.rb
+++ /dev/null
@@ -1,64 +0,0 @@
-# frozen_string_literal: true
-
-class StreamEntriesController < ApplicationController
-  include Authorization
-  include SignatureVerification
-
-  layout 'public'
-
-  before_action :set_account
-  before_action :set_stream_entry
-  before_action :set_link_headers
-  before_action :check_account_suspension
-  before_action :set_cache_headers
-
-  def show
-    respond_to do |format|
-      format.html do
-        expires_in 5.minutes, public: true unless @stream_entry.hidden?
-
-        redirect_to short_account_status_url(params[:account_username], @stream_entry.activity)
-      end
-
-      format.atom do
-        expires_in 3.minutes, public: true unless @stream_entry.hidden?
-
-        render xml: OStatus::AtomSerializer.render(OStatus::AtomSerializer.new.entry(@stream_entry, true))
-      end
-    end
-  end
-
-  def embed
-    redirect_to embed_short_account_status_url(@account, @stream_entry.activity), status: 301
-  end
-
-  private
-
-  def set_account
-    @account = Account.find_local!(params[:account_username])
-  end
-
-  def set_link_headers
-    response.headers['Link'] = LinkHeader.new(
-      [
-        [account_stream_entry_url(@account, @stream_entry, format: 'atom'), [%w(rel alternate), %w(type application/atom+xml)]],
-        [ActivityPub::TagManager.instance.uri_for(@stream_entry.activity), [%w(rel alternate), %w(type application/activity+json)]],
-      ]
-    )
-  end
-
-  def set_stream_entry
-    @stream_entry = @account.stream_entries.where(activity_type: 'Status').find(params[:id])
-    @type         = 'status'
-
-    raise ActiveRecord::RecordNotFound if @stream_entry.activity.nil?
-    authorize @stream_entry.activity, :show? if @stream_entry.hidden?
-  rescue Mastodon::NotPermittedError
-    # Reraise in order to get a 404
-    raise ActiveRecord::RecordNotFound
-  end
-
-  def check_account_suspension
-    gone if @account.suspended?
-  end
-end
diff --git a/app/helpers/admin/action_logs_helper.rb b/app/helpers/admin/action_logs_helper.rb
index e5fbb1500..1daa60774 100644
--- a/app/helpers/admin/action_logs_helper.rb
+++ b/app/helpers/admin/action_logs_helper.rb
@@ -89,7 +89,7 @@ module Admin::ActionLogsHelper
     when 'DomainBlock', 'EmailDomainBlock'
       link_to record.domain, "https://#{record.domain}"
     when 'Status'
-      link_to record.account.acct, TagManager.instance.url_for(record)
+      link_to record.account.acct, ActivityPub::TagManager.instance.url_for(record)
     when 'AccountWarning'
       link_to record.target_account.acct, admin_account_path(record.target_account_id)
     end
diff --git a/app/helpers/home_helper.rb b/app/helpers/home_helper.rb
index df60b7dd7..b66e827fe 100644
--- a/app/helpers/home_helper.rb
+++ b/app/helpers/home_helper.rb
@@ -21,7 +21,7 @@ module HomeHelper
                         end
                     end
                   else
-                    link_to(path || TagManager.instance.url_for(account), class: 'account__display-name') do
+                    link_to(path || ActivityPub::TagManager.instance.url_for(account), class: 'account__display-name') do
                       content_tag(:div, class: 'account__avatar-wrapper') do
                         content_tag(:div, '', class: 'account__avatar', style: "width: #{size}px; height: #{size}px; background-size: #{size}px #{size}px; background-image: url(#{full_asset_url(current_account&.user&.setting_auto_play_gif ? account.avatar_original_url : account.avatar_static_url)})")
                       end +
diff --git a/app/helpers/statuses_helper.rb b/app/helpers/statuses_helper.rb
new file mode 100644
index 000000000..e067380f6
--- /dev/null
+++ b/app/helpers/statuses_helper.rb
@@ -0,0 +1,222 @@
+# frozen_string_literal: true
+
+module StatusesHelper
+  EMBEDDED_CONTROLLER = 'statuses'
+  EMBEDDED_ACTION = 'embed'
+
+  def display_name(account, **options)
+    if options[:custom_emojify]
+      Formatter.instance.format_display_name(account, options)
+    else
+      account.display_name.presence || account.username
+    end
+  end
+
+  def account_action_button(account)
+    if user_signed_in?
+      if account.id == current_user.account_id
+        link_to settings_profile_url, class: 'button logo-button' do
+          safe_join([svg_logo, t('settings.edit_profile')])
+        end
+      elsif current_account.following?(account) || current_account.requested?(account)
+        link_to account_unfollow_path(account), class: 'button logo-button button--destructive', data: { method: :post } do
+          safe_join([svg_logo, t('accounts.unfollow')])
+        end
+      elsif !(account.memorial? || account.moved?)
+        link_to account_follow_path(account), class: "button logo-button#{account.blocking?(current_account) ? ' disabled' : ''}", data: { method: :post } do
+          safe_join([svg_logo, t('accounts.follow')])
+        end
+      end
+    elsif !(account.memorial? || account.moved?)
+      link_to account_remote_follow_path(account), class: 'button logo-button modal-button', target: '_new' do
+        safe_join([svg_logo, t('accounts.follow')])
+      end
+    end
+  end
+
+  def svg_logo
+    content_tag(:svg, tag(:use, 'xlink:href' => '#mastodon-svg-logo'), 'viewBox' => '0 0 216.4144 232.00976')
+  end
+
+  def svg_logo_full
+    content_tag(:svg, tag(:use, 'xlink:href' => '#mastodon-svg-logo-full'), 'viewBox' => '0 0 713.35878 175.8678')
+  end
+
+  def account_badge(account, all: false)
+    if account.bot?
+      content_tag(:div, content_tag(:div, t('accounts.roles.bot'), class: 'account-role bot'), class: 'roles')
+    elsif (Setting.show_staff_badge && account.user_staff?) || all
+      content_tag(:div, class: 'roles') do
+        if all && !account.user_staff?
+          content_tag(:div, t('admin.accounts.roles.user'), class: 'account-role')
+        elsif account.user_admin?
+          content_tag(:div, t('accounts.roles.admin'), class: 'account-role admin')
+        elsif account.user_moderator?
+          content_tag(:div, t('accounts.roles.moderator'), class: 'account-role moderator')
+        end
+      end
+    end
+  end
+
+  def link_to_more(url)
+    link_to t('statuses.show_more'), url, class: 'load-more load-gap'
+  end
+
+  def nothing_here(extra_classes = '')
+    content_tag(:div, class: "nothing-here #{extra_classes}") do
+      t('accounts.nothing_here')
+    end
+  end
+
+  def account_description(account)
+    prepend_str = [
+      [
+        number_to_human(account.statuses_count, strip_insignificant_zeros: true),
+        I18n.t('accounts.posts', count: account.statuses_count),
+      ].join(' '),
+
+      [
+        number_to_human(account.following_count, strip_insignificant_zeros: true),
+        I18n.t('accounts.following', count: account.following_count),
+      ].join(' '),
+
+      [
+        number_to_human(account.followers_count, strip_insignificant_zeros: true),
+        I18n.t('accounts.followers', count: account.followers_count),
+      ].join(' '),
+    ].join(', ')
+
+    [prepend_str, account.note].join(' · ')
+  end
+
+  def media_summary(status)
+    attachments = { image: 0, video: 0 }
+
+    status.media_attachments.each do |media|
+      if media.video?
+        attachments[:video] += 1
+      else
+        attachments[:image] += 1
+      end
+    end
+
+    text = attachments.to_a.reject { |_, value| value.zero? }.map { |key, value| I18n.t("statuses.attached.#{key}", count: value) }.join(' · ')
+
+    return if text.blank?
+
+    I18n.t('statuses.attached.description', attached: text)
+  end
+
+  def status_text_summary(status)
+    return if status.spoiler_text.blank?
+
+    I18n.t('statuses.content_warning', warning: status.spoiler_text)
+  end
+
+  def poll_summary(status)
+    return unless status.preloadable_poll
+
+    status.preloadable_poll.options.map { |o| "[ ] #{o}" }.join("\n")
+  end
+
+  def status_description(status)
+    components = [[media_summary(status), status_text_summary(status)].reject(&:blank?).join(' · ')]
+
+    if status.spoiler_text.blank?
+      components << status.text
+      components << poll_summary(status)
+    end
+
+    components.reject(&:blank?).join("\n\n")
+  end
+
+  def stream_link_target
+    embedded_view? ? '_blank' : nil
+  end
+
+  def acct(account)
+    if account.local?
+      "@#{account.acct}@#{Rails.configuration.x.local_domain}"
+    else
+      "@#{account.acct}"
+    end
+  end
+
+  def style_classes(status, is_predecessor, is_successor, include_threads)
+    classes = ['entry']
+    classes << 'entry-predecessor' if is_predecessor
+    classes << 'entry-reblog' if status.reblog?
+    classes << 'entry-successor' if is_successor
+    classes << 'entry-center' if include_threads
+    classes.join(' ')
+  end
+
+  def microformats_classes(status, is_direct_parent, is_direct_child)
+    classes = []
+    classes << 'p-in-reply-to' if is_direct_parent
+    classes << 'p-repost-of' if status.reblog? && is_direct_parent
+    classes << 'p-comment' if is_direct_child
+    classes.join(' ')
+  end
+
+  def microformats_h_class(status, is_predecessor, is_successor, include_threads)
+    if is_predecessor || status.reblog? || is_successor
+      'h-cite'
+    elsif include_threads
+      ''
+    else
+      'h-entry'
+    end
+  end
+
+  def rtl_status?(status)
+    status.local? ? rtl?(status.text) : rtl?(strip_tags(status.text))
+  end
+
+  def rtl?(text)
+    text = simplified_text(text)
+    rtl_words = text.scan(/[\p{Hebrew}\p{Arabic}\p{Syriac}\p{Thaana}\p{Nko}]+/m)
+
+    if rtl_words.present?
+      total_size = text.size.to_f
+      rtl_size(rtl_words) / total_size > 0.3
+    else
+      false
+    end
+  end
+
+  def fa_visibility_icon(status)
+    case status.visibility
+    when 'public'
+      fa_icon 'globe fw'
+    when 'unlisted'
+      fa_icon 'unlock fw'
+    when 'private'
+      fa_icon 'lock fw'
+    when 'direct'
+      fa_icon 'envelope fw'
+    end
+  end
+
+  private
+
+  def simplified_text(text)
+    text.dup.tap do |new_text|
+      URI.extract(new_text).each do |url|
+        new_text.gsub!(url, '')
+      end
+
+      new_text.gsub!(Account::MENTION_RE, '')
+      new_text.gsub!(Tag::HASHTAG_RE, '')
+      new_text.gsub!(/\s+/, '')
+    end
+  end
+
+  def rtl_size(words)
+    words.reduce(0) { |acc, elem| acc + elem.size }.to_f
+  end
+
+  def embedded_view?
+    params[:controller] == EMBEDDED_CONTROLLER && params[:action] == EMBEDDED_ACTION
+  end
+end
diff --git a/app/helpers/stream_entries_helper.rb b/app/helpers/stream_entries_helper.rb
deleted file mode 100644
index 02a860a74..000000000
--- a/app/helpers/stream_entries_helper.rb
+++ /dev/null
@@ -1,220 +0,0 @@
-# frozen_string_literal: true
-
-module StreamEntriesHelper
-  EMBEDDED_CONTROLLER = 'statuses'
-  EMBEDDED_ACTION = 'embed'
-
-  def display_name(account, **options)
-    if options[:custom_emojify]
-      Formatter.instance.format_display_name(account, options)
-    else
-      account.display_name.presence || account.username
-    end
-  end
-
-  def account_action_button(account)
-    if user_signed_in?
-      if account.id == current_user.account_id
-        link_to settings_profile_url, class: 'button logo-button' do
-          safe_join([svg_logo, t('settings.edit_profile')])
-        end
-      elsif current_account.following?(account) || current_account.requested?(account)
-        link_to account_unfollow_path(account), class: 'button logo-button button--destructive', data: { method: :post } do
-          safe_join([svg_logo, t('accounts.unfollow')])
-        end
-      elsif !(account.memorial? || account.moved?)
-        link_to account_follow_path(account), class: "button logo-button#{account.blocking?(current_account) ? ' disabled' : ''}", data: { method: :post } do
-          safe_join([svg_logo, t('accounts.follow')])
-        end
-      end
-    elsif !(account.memorial? || account.moved?)
-      link_to account_remote_follow_path(account), class: 'button logo-button modal-button', target: '_new' do
-        safe_join([svg_logo, t('accounts.follow')])
-      end
-    end
-  end
-
-  def svg_logo
-    content_tag(:svg, tag(:use, 'xlink:href' => '#mastodon-svg-logo'), 'viewBox' => '0 0 216.4144 232.00976')
-  end
-
-  def svg_logo_full
-    content_tag(:svg, tag(:use, 'xlink:href' => '#mastodon-svg-logo-full'), 'viewBox' => '0 0 713.35878 175.8678')
-  end
-
-  def account_badge(account, all: false)
-    if account.bot?
-      content_tag(:div, content_tag(:div, t('accounts.roles.bot'), class: 'account-role bot'), class: 'roles')
-    elsif (Setting.show_staff_badge && account.user_staff?) || all
-      content_tag(:div, class: 'roles') do
-        if all && !account.user_staff?
-          content_tag(:div, t('admin.accounts.roles.user'), class: 'account-role')
-        elsif account.user_admin?
-          content_tag(:div, t('accounts.roles.admin'), class: 'account-role admin')
-        elsif account.user_moderator?
-          content_tag(:div, t('accounts.roles.moderator'), class: 'account-role moderator')
-        end
-      end
-    end
-  end
-
-  def link_to_more(url)
-    link_to t('statuses.show_more'), url, class: 'load-more load-gap'
-  end
-
-  def nothing_here(extra_classes = '')
-    content_tag(:div, class: "nothing-here #{extra_classes}") do
-      t('accounts.nothing_here')
-    end
-  end
-
-  def account_description(account)
-    prepend_str = [
-      [
-        number_to_human(account.statuses_count, strip_insignificant_zeros: true),
-        I18n.t('accounts.posts', count: account.statuses_count),
-      ].join(' '),
-
-      [
-        number_to_human(account.following_count, strip_insignificant_zeros: true),
-        I18n.t('accounts.following', count: account.following_count),
-      ].join(' '),
-
-      [
-        number_to_human(account.followers_count, strip_insignificant_zeros: true),
-        I18n.t('accounts.followers', count: account.followers_count),
-      ].join(' '),
-    ].join(', ')
-
-    [prepend_str, account.note].join(' · ')
-  end
-
-  def media_summary(status)
-    attachments = { image: 0, video: 0 }
-
-    status.media_attachments.each do |media|
-      if media.video?
-        attachments[:video] += 1
-      else
-        attachments[:image] += 1
-      end
-    end
-
-    text = attachments.to_a.reject { |_, value| value.zero? }.map { |key, value| I18n.t("statuses.attached.#{key}", count: value) }.join(' · ')
-
-    return if text.blank?
-
-    I18n.t('statuses.attached.description', attached: text)
-  end
-
-  def status_text_summary(status)
-    return if status.spoiler_text.blank?
-    I18n.t('statuses.content_warning', warning: status.spoiler_text)
-  end
-
-  def poll_summary(status)
-    return unless status.preloadable_poll
-    status.preloadable_poll.options.map { |o| "[ ] #{o}" }.join("\n")
-  end
-
-  def status_description(status)
-    components = [[media_summary(status), status_text_summary(status)].reject(&:blank?).join(' · ')]
-
-    if status.spoiler_text.blank?
-      components << status.text
-      components << poll_summary(status)
-    end
-
-    components.reject(&:blank?).join("\n\n")
-  end
-
-  def stream_link_target
-    embedded_view? ? '_blank' : nil
-  end
-
-  def acct(account)
-    if account.local?
-      "@#{account.acct}@#{Rails.configuration.x.local_domain}"
-    else
-      "@#{account.acct}"
-    end
-  end
-
-  def style_classes(status, is_predecessor, is_successor, include_threads)
-    classes = ['entry']
-    classes << 'entry-predecessor' if is_predecessor
-    classes << 'entry-reblog' if status.reblog?
-    classes << 'entry-successor' if is_successor
-    classes << 'entry-center' if include_threads
-    classes.join(' ')
-  end
-
-  def microformats_classes(status, is_direct_parent, is_direct_child)
-    classes = []
-    classes << 'p-in-reply-to' if is_direct_parent
-    classes << 'p-repost-of' if status.reblog? && is_direct_parent
-    classes << 'p-comment' if is_direct_child
-    classes.join(' ')
-  end
-
-  def microformats_h_class(status, is_predecessor, is_successor, include_threads)
-    if is_predecessor || status.reblog? || is_successor
-      'h-cite'
-    elsif include_threads
-      ''
-    else
-      'h-entry'
-    end
-  end
-
-  def rtl_status?(status)
-    status.local? ? rtl?(status.text) : rtl?(strip_tags(status.text))
-  end
-
-  def rtl?(text)
-    text = simplified_text(text)
-    rtl_words = text.scan(/[\p{Hebrew}\p{Arabic}\p{Syriac}\p{Thaana}\p{Nko}]+/m)
-
-    if rtl_words.present?
-      total_size = text.size.to_f
-      rtl_size(rtl_words) / total_size > 0.3
-    else
-      false
-    end
-  end
-
-  def fa_visibility_icon(status)
-    case status.visibility
-    when 'public'
-      fa_icon 'globe fw'
-    when 'unlisted'
-      fa_icon 'unlock fw'
-    when 'private'
-      fa_icon 'lock fw'
-    when 'direct'
-      fa_icon 'envelope fw'
-    end
-  end
-
-  private
-
-  def simplified_text(text)
-    text.dup.tap do |new_text|
-      URI.extract(new_text).each do |url|
-        new_text.gsub!(url, '')
-      end
-
-      new_text.gsub!(Account::MENTION_RE, '')
-      new_text.gsub!(Tag::HASHTAG_RE, '')
-      new_text.gsub!(/\s+/, '')
-    end
-  end
-
-  def rtl_size(words)
-    words.reduce(0) { |acc, elem| acc + elem.size }.to_f
-  end
-
-  def embedded_view?
-    params[:controller] == EMBEDDED_CONTROLLER && params[:action] == EMBEDDED_ACTION
-  end
-end
diff --git a/app/javascript/styles/application.scss b/app/javascript/styles/application.scss
index 6db3bc3dc..8ebc45b62 100644
--- a/app/javascript/styles/application.scss
+++ b/app/javascript/styles/application.scss
@@ -13,7 +13,7 @@
 @import 'mastodon/widgets';
 @import 'mastodon/forms';
 @import 'mastodon/accounts';
-@import 'mastodon/stream_entries';
+@import 'mastodon/statuses';
 @import 'mastodon/boost';
 @import 'mastodon/components';
 @import 'mastodon/polls';
diff --git a/app/javascript/styles/mastodon/statuses.scss b/app/javascript/styles/mastodon/statuses.scss
new file mode 100644
index 000000000..19ce0ab8f
--- /dev/null
+++ b/app/javascript/styles/mastodon/statuses.scss
@@ -0,0 +1,163 @@
+.activity-stream {
+  box-shadow: 0 0 15px rgba($base-shadow-color, 0.2);
+  border-radius: 4px;
+  overflow: hidden;
+  margin-bottom: 10px;
+
+  &--under-tabs {
+    border-radius: 0 0 4px 4px;
+  }
+
+  @media screen and (max-width: $no-gap-breakpoint) {
+    margin-bottom: 0;
+    border-radius: 0;
+    box-shadow: none;
+  }
+
+  &--headless {
+    border-radius: 0;
+    margin: 0;
+    box-shadow: none;
+
+    .detailed-status,
+    .status {
+      border-radius: 0 !important;
+    }
+  }
+
+  div[data-component] {
+    width: 100%;
+  }
+
+  .entry {
+    background: $ui-base-color;
+
+    .detailed-status,
+    .status,
+    .load-more {
+      animation: none;
+    }
+
+    &:last-child {
+      .detailed-status,
+      .status,
+      .load-more {
+        border-bottom: 0;
+        border-radius: 0 0 4px 4px;
+      }
+    }
+
+    &:first-child {
+      .detailed-status,
+      .status,
+      .load-more {
+        border-radius: 4px 4px 0 0;
+      }
+
+      &:last-child {
+        .detailed-status,
+        .status,
+        .load-more {
+          border-radius: 4px;
+        }
+      }
+    }
+
+    @media screen and (max-width: 740px) {
+      .detailed-status,
+      .status,
+      .load-more {
+        border-radius: 0 !important;
+      }
+    }
+  }
+
+  &--highlighted .entry {
+    background: lighten($ui-base-color, 8%);
+  }
+}
+
+.button.logo-button {
+  flex: 0 auto;
+  font-size: 14px;
+  background: $ui-highlight-color;
+  color: $primary-text-color;
+  text-transform: none;
+  line-height: 36px;
+  height: auto;
+  padding: 3px 15px;
+  border: 0;
+
+  svg {
+    width: 20px;
+    height: auto;
+    vertical-align: middle;
+    margin-right: 5px;
+    fill: $primary-text-color;
+  }
+
+  &:active,
+  &:focus,
+  &:hover {
+    background: lighten($ui-highlight-color, 10%);
+  }
+
+  &:disabled,
+  &.disabled {
+    &:active,
+    &:focus,
+    &:hover {
+      background: $ui-primary-color;
+    }
+  }
+
+  &.button--destructive {
+    &:active,
+    &:focus,
+    &:hover {
+      background: $error-red;
+    }
+  }
+
+  @media screen and (max-width: $no-gap-breakpoint) {
+    svg {
+      display: none;
+    }
+  }
+}
+
+.embed,
+.public-layout {
+  .detailed-status {
+    padding: 15px;
+  }
+
+  .status {
+    padding: 15px 15px 15px (48px + 15px * 2);
+    min-height: 48px + 2px;
+
+    &__avatar {
+      left: 15px;
+      top: 17px;
+    }
+
+    &__content {
+      padding-top: 5px;
+    }
+
+    &__prepend {
+      margin-left: 48px + 15px * 2;
+      padding-top: 15px;
+    }
+
+    &__prepend-icon-wrapper {
+      left: -32px;
+    }
+
+    .media-gallery,
+    &__action-bar,
+    .video-player {
+      margin-top: 10px;
+    }
+  }
+}
diff --git a/app/javascript/styles/mastodon/stream_entries.scss b/app/javascript/styles/mastodon/stream_entries.scss
deleted file mode 100644
index 19ce0ab8f..000000000
--- a/app/javascript/styles/mastodon/stream_entries.scss
+++ /dev/null
@@ -1,163 +0,0 @@
-.activity-stream {
-  box-shadow: 0 0 15px rgba($base-shadow-color, 0.2);
-  border-radius: 4px;
-  overflow: hidden;
-  margin-bottom: 10px;
-
-  &--under-tabs {
-    border-radius: 0 0 4px 4px;
-  }
-
-  @media screen and (max-width: $no-gap-breakpoint) {
-    margin-bottom: 0;
-    border-radius: 0;
-    box-shadow: none;
-  }
-
-  &--headless {
-    border-radius: 0;
-    margin: 0;
-    box-shadow: none;
-
-    .detailed-status,
-    .status {
-      border-radius: 0 !important;
-    }
-  }
-
-  div[data-component] {
-    width: 100%;
-  }
-
-  .entry {
-    background: $ui-base-color;
-
-    .detailed-status,
-    .status,
-    .load-more {
-      animation: none;
-    }
-
-    &:last-child {
-      .detailed-status,
-      .status,
-      .load-more {
-        border-bottom: 0;
-        border-radius: 0 0 4px 4px;
-      }
-    }
-
-    &:first-child {
-      .detailed-status,
-      .status,
-      .load-more {
-        border-radius: 4px 4px 0 0;
-      }
-
-      &:last-child {
-        .detailed-status,
-        .status,
-        .load-more {
-          border-radius: 4px;
-        }
-      }
-    }
-
-    @media screen and (max-width: 740px) {
-      .detailed-status,
-      .status,
-      .load-more {
-        border-radius: 0 !important;
-      }
-    }
-  }
-
-  &--highlighted .entry {
-    background: lighten($ui-base-color, 8%);
-  }
-}
-
-.button.logo-button {
-  flex: 0 auto;
-  font-size: 14px;
-  background: $ui-highlight-color;
-  color: $primary-text-color;
-  text-transform: none;
-  line-height: 36px;
-  height: auto;
-  padding: 3px 15px;
-  border: 0;
-
-  svg {
-    width: 20px;
-    height: auto;
-    vertical-align: middle;
-    margin-right: 5px;
-    fill: $primary-text-color;
-  }
-
-  &:active,
-  &:focus,
-  &:hover {
-    background: lighten($ui-highlight-color, 10%);
-  }
-
-  &:disabled,
-  &.disabled {
-    &:active,
-    &:focus,
-    &:hover {
-      background: $ui-primary-color;
-    }
-  }
-
-  &.button--destructive {
-    &:active,
-    &:focus,
-    &:hover {
-      background: $error-red;
-    }
-  }
-
-  @media screen and (max-width: $no-gap-breakpoint) {
-    svg {
-      display: none;
-    }
-  }
-}
-
-.embed,
-.public-layout {
-  .detailed-status {
-    padding: 15px;
-  }
-
-  .status {
-    padding: 15px 15px 15px (48px + 15px * 2);
-    min-height: 48px + 2px;
-
-    &__avatar {
-      left: 15px;
-      top: 17px;
-    }
-
-    &__content {
-      padding-top: 5px;
-    }
-
-    &__prepend {
-      margin-left: 48px + 15px * 2;
-      padding-top: 15px;
-    }
-
-    &__prepend-icon-wrapper {
-      left: -32px;
-    }
-
-    .media-gallery,
-    &__action-bar,
-    .video-player {
-      margin-top: 10px;
-    }
-  }
-}
diff --git a/app/lib/formatter.rb b/app/lib/formatter.rb
index 8a1aad41a..6c1239963 100644
--- a/app/lib/formatter.rb
+++ b/app/lib/formatter.rb
@@ -295,6 +295,6 @@ class Formatter
   end
 
   def mention_html(account)
-    "<span class=\"h-card\"><a href=\"#{encode(TagManager.instance.url_for(account))}\" class=\"u-url mention\">@<span>#{encode(account.username)}</span></a></span>"
+    "<span class=\"h-card\"><a href=\"#{encode(ActivityPub::TagManager.instance.url_for(account))}\" class=\"u-url mention\">@<span>#{encode(account.username)}</span></a></span>"
   end
 end
diff --git a/app/lib/ostatus/atom_serializer.rb b/app/lib/ostatus/atom_serializer.rb
deleted file mode 100644
index f5c0e85ca..000000000
--- a/app/lib/ostatus/atom_serializer.rb
+++ /dev/null
@@ -1,376 +0,0 @@
-# frozen_string_literal: true
-
-class OStatus::AtomSerializer
-  include RoutingHelper
-  include ActionView::Helpers::SanitizeHelper
-
-  class << self
-    def render(element)
-      document = Ox::Document.new(version: '1.0')
-      document << element
-      ('<?xml version="1.0"?>' + Ox.dump(element, effort: :tolerant)).force_encoding('UTF-8')
-    end
-  end
-
-  def author(account)
-    author = Ox::Element.new('author')
-
-    uri = OStatus::TagManager.instance.uri_for(account)
-
-    append_element(author, 'id', uri)
-    append_element(author, 'activity:object-type', OStatus::TagManager::TYPES[:person])
-    append_element(author, 'uri', uri)
-    append_element(author, 'name', account.username)
-    append_element(author, 'email', account.local? ? account.local_username_and_domain : account.acct)
-    append_element(author, 'summary', Formatter.instance.simplified_format(account).to_str, type: :html) if account.note?
-    append_element(author, 'link', nil, rel: :alternate, type: 'text/html', href: ::TagManager.instance.url_for(account))
-    append_element(author, 'link', nil, rel: :avatar, type: account.avatar_content_type, 'media:width': 120, 'media:height': 120, href: full_asset_url(account.avatar.url(:original))) if account.avatar?
-    append_element(author, 'link', nil, rel: :header, type: account.header_content_type, 'media:width': 700, 'media:height': 335, href: full_asset_url(account.header.url(:original))) if account.header?
-    account.emojis.each do |emoji|
-      append_element(author, 'link', nil, rel: :emoji, href: full_asset_url(emoji.image.url), name: emoji.shortcode)
-    end
-    append_element(author, 'poco:preferredUsername', account.username)
-    append_element(author, 'poco:displayName', account.display_name) if account.display_name?
-    append_element(author, 'poco:note', account.local? ? account.note : strip_tags(account.note)) if account.note?
-    append_element(author, 'mastodon:scope', account.locked? ? :private : :public)
-
-    author
-  end
-
-  def feed(account, stream_entries)
-    feed = Ox::Element.new('feed')
-
-    add_namespaces(feed)
-
-    append_element(feed, 'id', account_url(account, format: 'atom'))
-    append_element(feed, 'title', account.display_name.presence || account.username)
-    append_element(feed, 'subtitle', account.note)
-    append_element(feed, 'updated', account.updated_at.iso8601)
-    append_element(feed, 'logo', full_asset_url(account.avatar.url(:original)))
-
-    feed << author(account)
-
-    append_element(feed, 'link', nil, rel: :alternate, type: 'text/html', href: ::TagManager.instance.url_for(account))
-    append_element(feed, 'link', nil, rel: :self, type: 'application/atom+xml', href: account_url(account, format: 'atom'))
-    append_element(feed, 'link', nil, rel: :next, type: 'application/atom+xml', href: account_url(account, format: 'atom', max_id: stream_entries.last.id)) if stream_entries.size == 20
-
-    stream_entries.each do |stream_entry|
-      feed << entry(stream_entry)
-    end
-
-    feed
-  end
-
-  def entry(stream_entry, root = false)
-    entry = Ox::Element.new('entry')
-
-    add_namespaces(entry) if root
-
-    append_element(entry, 'id', OStatus::TagManager.instance.uri_for(stream_entry.status))
-    append_element(entry, 'published', stream_entry.created_at.iso8601)
-    append_element(entry, 'updated', stream_entry.updated_at.iso8601)
-    append_element(entry, 'title', stream_entry&.status&.title || "#{stream_entry.account.acct} deleted status")
-
-    entry << author(stream_entry.account) if root
-
-    append_element(entry, 'activity:object-type', OStatus::TagManager::TYPES[stream_entry.object_type])
-    append_element(entry, 'activity:verb', OStatus::TagManager::VERBS[stream_entry.verb])
-
-    entry << object(stream_entry.target) if stream_entry.targeted?
-
-    if stream_entry.status.nil?
-      append_element(entry, 'content', 'Deleted status')
-    elsif stream_entry.status.destroyed?
-      append_element(entry, 'content', 'Deleted status')
-      append_element(entry, 'link', nil, rel: :alternate, type: 'application/activity+json', href: ActivityPub::TagManager.instance.uri_for(stream_entry.status)) if stream_entry.account.local?
-    else
-      serialize_status_attributes(entry, stream_entry.status)
-    end
-
-    append_element(entry, 'link', nil, rel: :alternate, type: 'text/html', href: ::TagManager.instance.url_for(stream_entry.status))
-    append_element(entry, 'link', nil, rel: :self, type: 'application/atom+xml', href: account_stream_entry_url(stream_entry.account, stream_entry, format: 'atom'))
-    append_element(entry, 'thr:in-reply-to', nil, ref: OStatus::TagManager.instance.uri_for(stream_entry.thread), href: ::TagManager.instance.url_for(stream_entry.thread)) if stream_entry.threaded?
-    append_element(entry, 'ostatus:conversation', nil, ref: conversation_uri(stream_entry.status.conversation)) unless stream_entry&.status&.conversation_id.nil?
-
-    entry
-  end
-
-  def object(status)
-    object = Ox::Element.new('activity:object')
-
-    append_element(object, 'id', OStatus::TagManager.instance.uri_for(status))
-    append_element(object, 'published', status.created_at.iso8601)
-    append_element(object, 'updated', status.updated_at.iso8601)
-    append_element(object, 'title', status.title)
-
-    object << author(status.account)
-
-    append_element(object, 'activity:object-type', OStatus::TagManager::TYPES[status.object_type])
-    append_element(object, 'activity:verb', OStatus::TagManager::VERBS[status.verb])
-
-    serialize_status_attributes(object, status)
-
-    append_element(object, 'link', nil, rel: :alternate, type: 'text/html', href: ::TagManager.instance.url_for(status))
-    append_element(object, 'thr:in-reply-to', nil, ref: OStatus::TagManager.instance.uri_for(status.thread), href: ::TagManager.instance.url_for(status.thread)) unless status.thread.nil?
-    append_element(object, 'ostatus:conversation', nil, ref: conversation_uri(status.conversation)) unless status.conversation_id.nil?
-
-    object
-  end
-
-  def follow_salmon(follow)
-    entry = Ox::Element.new('entry')
-    add_namespaces(entry)
-
-    description = "#{follow.account.acct} started following #{follow.target_account.acct}"
-
-    append_element(entry, 'id', OStatus::TagManager.instance.unique_tag(follow.created_at, follow.id, 'Follow'))
-    append_element(entry, 'title', description)
-    append_element(entry, 'content', description, type: :html)
-
-    entry << author(follow.account)
-
-    append_element(entry, 'activity:object-type', OStatus::TagManager::TYPES[:activity])
-    append_element(entry, 'activity:verb', OStatus::TagManager::VERBS[:follow])
-
-    object = author(follow.target_account)
-    object.value = 'activity:object'
-
-    entry << object
-    entry
-  end
-
-  def follow_request_salmon(follow_request)
-    entry = Ox::Element.new('entry')
-    add_namespaces(entry)
-
-    append_element(entry, 'id', OStatus::TagManager.instance.unique_tag(follow_request.created_at, follow_request.id, 'FollowRequest'))
-    append_element(entry, 'title', "#{follow_request.account.acct} requested to follow #{follow_request.target_account.acct}")
-
-    entry << author(follow_request.account)
-
-    append_element(entry, 'activity:object-type', OStatus::TagManager::TYPES[:activity])
-    append_element(entry, 'activity:verb', OStatus::TagManager::VERBS[:request_friend])
-
-    object = author(follow_request.target_account)
-    object.value = 'activity:object'
-
-    entry << object
-    entry
-  end
-
-  def authorize_follow_request_salmon(follow_request)
-    entry = Ox::Element.new('entry')
-    add_namespaces(entry)
-
-    append_element(entry, 'id', OStatus::TagManager.instance.unique_tag(Time.now.utc, follow_request.id, 'FollowRequest'))
-    append_element(entry, 'title', "#{follow_request.target_account.acct} authorizes follow request by #{follow_request.account.acct}")
-
-    entry << author(follow_request.target_account)
-
-    append_element(entry, 'activity:object-type', OStatus::TagManager::TYPES[:activity])
-    append_element(entry, 'activity:verb', OStatus::TagManager::VERBS[:authorize])
-
-    object = Ox::Element.new('activity:object')
-    object << author(follow_request.account)
-
-    append_element(object, 'activity:object-type', OStatus::TagManager::TYPES[:activity])
-    append_element(object, 'activity:verb', OStatus::TagManager::VERBS[:request_friend])
-
-    inner_object = author(follow_request.target_account)
-    inner_object.value = 'activity:object'
-
-    object << inner_object
-    entry  << object
-    entry
-  end
-
-  def reject_follow_request_salmon(follow_request)
-    entry = Ox::Element.new('entry')
-    add_namespaces(entry)
-
-    append_element(entry, 'id', OStatus::TagManager.instance.unique_tag(Time.now.utc, follow_request.id, 'FollowRequest'))
-    append_element(entry, 'title', "#{follow_request.target_account.acct} rejects follow request by #{follow_request.account.acct}")
-
-    entry << author(follow_request.target_account)
-
-    append_element(entry, 'activity:object-type', OStatus::TagManager::TYPES[:activity])
-    append_element(entry, 'activity:verb', OStatus::TagManager::VERBS[:reject])
-
-    object = Ox::Element.new('activity:object')
-    object << author(follow_request.account)
-
-    append_element(object, 'activity:object-type', OStatus::TagManager::TYPES[:activity])
-    append_element(object, 'activity:verb', OStatus::TagManager::VERBS[:request_friend])
-
-    inner_object = author(follow_request.target_account)
-    inner_object.value = 'activity:object'
-
-    object << inner_object
-    entry  << object
-    entry
-  end
-
-  def unfollow_salmon(follow)
-    entry = Ox::Element.new('entry')
-    add_namespaces(entry)
-
-    description = "#{follow.account.acct} is no longer following #{follow.target_account.acct}"
-
-    append_element(entry, 'id', OStatus::TagManager.instance.unique_tag(Time.now.utc, follow.id, 'Follow'))
-    append_element(entry, 'title', description)
-    append_element(entry, 'content', description, type: :html)
-
-    entry << author(follow.account)
-
-    append_element(entry, 'activity:object-type', OStatus::TagManager::TYPES[:activity])
-    append_element(entry, 'activity:verb', OStatus::TagManager::VERBS[:unfollow])
-
-    object = author(follow.target_account)
-    object.value = 'activity:object'
-
-    entry << object
-    entry
-  end
-
-  def block_salmon(block)
-    entry = Ox::Element.new('entry')
-    add_namespaces(entry)
-
-    description = "#{block.account.acct} no longer wishes to interact with #{block.target_account.acct}"
-
-    append_element(entry, 'id', OStatus::TagManager.instance.unique_tag(Time.now.utc, block.id, 'Block'))
-    append_element(entry, 'title', description)
-
-    entry << author(block.account)
-
-    append_element(entry, 'activity:object-type', OStatus::TagManager::TYPES[:activity])
-    append_element(entry, 'activity:verb', OStatus::TagManager::VERBS[:block])
-
-    object = author(block.target_account)
-    object.value = 'activity:object'
-
-    entry << object
-    entry
-  end
-
-  def unblock_salmon(block)
-    entry = Ox::Element.new('entry')
-    add_namespaces(entry)
-
-    description = "#{block.account.acct} no longer blocks #{block.target_account.acct}"
-
-    append_element(entry, 'id', OStatus::TagManager.instance.unique_tag(Time.now.utc, block.id, 'Block'))
-    append_element(entry, 'title', description)
-
-    entry << author(block.account)
-
-    append_element(entry, 'activity:object-type', OStatus::TagManager::TYPES[:activity])
-    append_element(entry, 'activity:verb', OStatus::TagManager::VERBS[:unblock])
-
-    object = author(block.target_account)
-    object.value = 'activity:object'
-
-    entry << object
-    entry
-  end
-
-  def favourite_salmon(favourite)
-    entry = Ox::Element.new('entry')
-    add_namespaces(entry)
-
-    description = "#{favourite.account.acct} favourited a status by #{favourite.status.account.acct}"
-
-    append_element(entry, 'id', OStatus::TagManager.instance.unique_tag(favourite.created_at, favourite.id, 'Favourite'))
-    append_element(entry, 'title', description)
-    append_element(entry, 'content', description, type: :html)
-
-    entry << author(favourite.account)
-
-    append_element(entry, 'activity:object-type', OStatus::TagManager::TYPES[:activity])
-    append_element(entry, 'activity:verb', OStatus::TagManager::VERBS[:favorite])
-
-    entry << object(favourite.status)
-
-    append_element(entry, 'thr:in-reply-to', nil, ref: OStatus::TagManager.instance.uri_for(favourite.status), href: ::TagManager.instance.url_for(favourite.status))
-
-    entry
-  end
-
-  def unfavourite_salmon(favourite)
-    entry = Ox::Element.new('entry')
-    add_namespaces(entry)
-
-    description = "#{favourite.account.acct} no longer favourites a status by #{favourite.status.account.acct}"
-
-    append_element(entry, 'id', OStatus::TagManager.instance.unique_tag(Time.now.utc, favourite.id, 'Favourite'))
-    append_element(entry, 'title', description)
-    append_element(entry, 'content', description, type: :html)
-
-    entry << author(favourite.account)
-
-    append_element(entry, 'activity:object-type', OStatus::TagManager::TYPES[:activity])
-    append_element(entry, 'activity:verb', OStatus::TagManager::VERBS[:unfavorite])
-
-    entry << object(favourite.status)
-
-    append_element(entry, 'thr:in-reply-to', nil, ref: OStatus::TagManager.instance.uri_for(favourite.status), href: ::TagManager.instance.url_for(favourite.status))
-
-    entry
-  end
-
-  private
-
-  def append_element(parent, name, content = nil, **attributes)
-    element = Ox::Element.new(name)
-    attributes.each { |k, v| element[k] = sanitize_str(v) }
-    element << sanitize_str(content) unless content.nil?
-    parent  << element
-  end
-
-  def sanitize_str(raw_str)
-    raw_str.to_s
-  end
-
-  def conversation_uri(conversation)
-    return conversation.uri if conversation.uri?
-    OStatus::TagManager.instance.unique_tag(conversation.created_at, conversation.id, 'Conversation')
-  end
-
-  def add_namespaces(parent)
-    parent['xmlns']          = OStatus::TagManager::XMLNS
-    parent['xmlns:thr']      = OStatus::TagManager::THR_XMLNS
-    parent['xmlns:activity'] = OStatus::TagManager::AS_XMLNS
-    parent['xmlns:poco']     = OStatus::TagManager::POCO_XMLNS
-    parent['xmlns:media']    = OStatus::TagManager::MEDIA_XMLNS
-    parent['xmlns:ostatus']  = OStatus::TagManager::OS_XMLNS
-    parent['xmlns:mastodon'] = OStatus::TagManager::MTDN_XMLNS
-  end
-
-  def serialize_status_attributes(entry, status)
-    append_element(entry, 'link', nil, rel: :alternate, type: 'application/activity+json', href: ActivityPub::TagManager.instance.uri_for(status)) if status.account.local?
-
-    append_element(entry, 'summary', status.spoiler_text, 'xml:lang': status.language) if status.spoiler_text?
-    append_element(entry, 'content', Formatter.instance.format(status, inline_poll_options: true).to_str || '.', type: 'html', 'xml:lang': status.language)
-
-    status.active_mentions.sort_by(&:id).each do |mentioned|
-      append_element(entry, 'link', nil, rel: :mentioned, 'ostatus:object-type': OStatus::TagManager::TYPES[:person], href: OStatus::TagManager.instance.uri_for(mentioned.account))
-    end
-
-    append_element(entry, 'link', nil, rel: :mentioned, 'ostatus:object-type': OStatus::TagManager::TYPES[:collection], href: OStatus::TagManager::COLLECTIONS[:public]) if status.public_visibility?
-
-    status.tags.each do |tag|
-      append_element(entry, 'category', nil, term: tag.name)
-    end
-
-    status.media_attachments.each do |media|
-      append_element(entry, 'link', nil, rel: :enclosure, type: media.file_content_type, length: media.file_file_size, href: full_asset_url(media.file.url(:original, false)))
-    end
-
-    append_element(entry, 'category', nil, term: 'nsfw') if status.sensitive? && status.media_attachments.any?
-    append_element(entry, 'mastodon:scope', status.visibility)
-
-    status.emojis.each do |emoji|
-      append_element(entry, 'link', nil, rel: :emoji, href: full_asset_url(emoji.image.url), name: emoji.shortcode)
-    end
-  end
-end
diff --git a/app/lib/status_finder.rb b/app/lib/status_finder.rb
index 4d1aed297..22ced8bf8 100644
--- a/app/lib/status_finder.rb
+++ b/app/lib/status_finder.rb
@@ -13,8 +13,6 @@ class StatusFinder
     raise ActiveRecord::RecordNotFound unless TagManager.instance.local_url?(url)
 
     case recognized_params[:controller]
-    when 'stream_entries'
-      StreamEntry.find(recognized_params[:id]).status
     when 'statuses'
       Status.find(recognized_params[:id])
     else
diff --git a/app/lib/tag_manager.rb b/app/lib/tag_manager.rb
index fb364cb98..daf4f556b 100644
--- a/app/lib/tag_manager.rb
+++ b/app/lib/tag_manager.rb
@@ -33,15 +33,4 @@ class TagManager
     domain = uri.host + (uri.port ? ":#{uri.port}" : '')
     TagManager.instance.web_domain?(domain)
   end
-
-  def url_for(target)
-    return target.url if target.respond_to?(:local?) && !target.local?
-
-    case target.object_type
-    when :person
-      short_account_url(target)
-    when :note, :comment, :activity
-      short_account_status_url(target.account, target)
-    end
-  end
 end
diff --git a/app/mailers/admin_mailer.rb b/app/mailers/admin_mailer.rb
index db154cad5..9ab3e2bbd 100644
--- a/app/mailers/admin_mailer.rb
+++ b/app/mailers/admin_mailer.rb
@@ -3,7 +3,7 @@
 class AdminMailer < ApplicationMailer
   layout 'plain_mailer'
 
-  helper :stream_entries
+  helper :statuses
 
   def new_report(recipient, report)
     @report   = report
diff --git a/app/mailers/notification_mailer.rb b/app/mailers/notification_mailer.rb
index 66fa337c1..723d901fc 100644
--- a/app/mailers/notification_mailer.rb
+++ b/app/mailers/notification_mailer.rb
@@ -1,7 +1,7 @@
 # frozen_string_literal: true
 
 class NotificationMailer < ApplicationMailer
-  helper :stream_entries
+  helper :statuses
 
   add_template_helper RoutingHelper
 
diff --git a/app/models/concerns/account_associations.rb b/app/models/concerns/account_associations.rb
index 70855e054..0921e3252 100644
--- a/app/models/concerns/account_associations.rb
+++ b/app/models/concerns/account_associations.rb
@@ -11,7 +11,6 @@ module AccountAssociations
     has_many :identity_proofs, class_name: 'AccountIdentityProof', dependent: :destroy, inverse_of: :account
 
     # Timelines
-    has_many :stream_entries, inverse_of: :account, dependent: :destroy
     has_many :statuses, inverse_of: :account, dependent: :destroy
     has_many :favourites, inverse_of: :account, dependent: :destroy
     has_many :mentions, inverse_of: :account, dependent: :destroy
diff --git a/app/models/concerns/streamable.rb b/app/models/concerns/streamable.rb
deleted file mode 100644
index 7c9edb8ef..000000000
--- a/app/models/concerns/streamable.rb
+++ /dev/null
@@ -1,43 +0,0 @@
-# frozen_string_literal: true
-
-module Streamable
-  extend ActiveSupport::Concern
-
-  included do
-    has_one :stream_entry, as: :activity
-
-    after_create do
-      account.stream_entries.create!(activity: self, hidden: hidden?) if needs_stream_entry?
-    end
-  end
-
-  def title
-    super
-  end
-
-  def content
-    title
-  end
-
-  def target
-    super
-  end
-
-  def object_type
-    :activity
-  end
-
-  def thread
-    super
-  end
-
-  def hidden?
-    false
-  end
-
-  private
-
-  def needs_stream_entry?
-    account.local?
-  end
-end
diff --git a/app/models/remote_profile.rb b/app/models/remote_profile.rb
deleted file mode 100644
index 742d2b56f..000000000
--- a/app/models/remote_profile.rb
+++ /dev/null
@@ -1,57 +0,0 @@
-# frozen_string_literal: true
-
-class RemoteProfile
-  include ActiveModel::Model
-
-  attr_reader :document
-
-  def initialize(body)
-    @document = Nokogiri::XML.parse(body, nil, 'utf-8')
-  end
-
-  def root
-    @root ||= document.at_xpath('/atom:feed|/atom:entry', atom: OStatus::TagManager::XMLNS)
-  end
-
-  def author
-    @author ||= root.at_xpath('./atom:author|./dfrn:owner', atom: OStatus::TagManager::XMLNS, dfrn: OStatus::TagManager::DFRN_XMLNS)
-  end
-
-  def hub_link
-    @hub_link ||= link_href_from_xml(root, 'hub')
-  end
-
-  def display_name
-    @display_name ||= author.at_xpath('./poco:displayName', poco: OStatus::TagManager::POCO_XMLNS)&.content
-  end
-
-  def note
-    @note ||= author.at_xpath('./atom:summary|./poco:note', atom: OStatus::TagManager::XMLNS, poco: OStatus::TagManager::POCO_XMLNS)&.content
-  end
-
-  def scope
-    @scope ||= author.at_xpath('./mastodon:scope', mastodon: OStatus::TagManager::MTDN_XMLNS)&.content
-  end
-
-  def avatar
-    @avatar ||= link_href_from_xml(author, 'avatar')
-  end
-
-  def header
-    @header ||= link_href_from_xml(author, 'header')
-  end
-
-  def emojis
-    @emojis ||= author.xpath('./xmlns:link[@rel="emoji"]', xmlns: OStatus::TagManager::XMLNS)
-  end
-
-  def locked?
-    scope == 'private'
-  end
-
-  private
-
-  def link_href_from_xml(xml, type)
-    xml.at_xpath(%(./atom:link[@rel="#{type}"]/@href), atom: OStatus::TagManager::XMLNS)&.content
-  end
-end
diff --git a/app/models/status.rb b/app/models/status.rb
index 2258e2d07..906756e85 100644
--- a/app/models/status.rb
+++ b/app/models/status.rb
@@ -28,7 +28,6 @@ class Status < ApplicationRecord
   before_destroy :unlink_from_conversations
 
   include Paginable
-  include Streamable
   include Cacheable
   include StatusThreadingConcern
 
@@ -61,7 +60,6 @@ class Status < ApplicationRecord
   has_and_belongs_to_many :preview_cards
 
   has_one :notification, as: :activity, dependent: :destroy
-  has_one :stream_entry, as: :activity, inverse_of: :status
   has_one :status_stat, inverse_of: :status
   has_one :poll, inverse_of: :status, dependent: :destroy
 
@@ -106,13 +104,11 @@ class Status < ApplicationRecord
                    :status_stat,
                    :tags,
                    :preview_cards,
-                   :stream_entry,
                    :preloadable_poll,
                    account: :account_stat,
                    active_mentions: { account: :account_stat },
                    reblog: [
                      :application,
-                     :stream_entry,
                      :tags,
                      :preview_cards,
                      :media_attachments,
diff --git a/app/models/stream_entry.rb b/app/models/stream_entry.rb
deleted file mode 100644
index 1a9afc5c7..000000000
--- a/app/models/stream_entry.rb
+++ /dev/null
@@ -1,59 +0,0 @@
-# frozen_string_literal: true
-# == Schema Information
-#
-# Table name: stream_entries
-#
-#  id            :bigint(8)        not null, primary key
-#  activity_id   :bigint(8)
-#  activity_type :string
-#  created_at    :datetime         not null
-#  updated_at    :datetime         not null
-#  hidden        :boolean          default(FALSE), not null
-#  account_id    :bigint(8)
-#
-
-class StreamEntry < ApplicationRecord
-  include Paginable
-
-  belongs_to :account, inverse_of: :stream_entries
-  belongs_to :activity, polymorphic: true
-  belongs_to :status, foreign_type: 'Status', foreign_key: 'activity_id', inverse_of: :stream_entry
-
-  validates :account, :activity, presence: true
-
-  STATUS_INCLUDES = [:account, :stream_entry, :conversation, :media_attachments, :tags, mentions: :account, reblog: [:stream_entry, :account, :conversation, :media_attachments, :tags, mentions: :account], thread: [:stream_entry, :account]].freeze
-
-  default_scope { where(activity_type: 'Status') }
-  scope :recent, -> { reorder(id: :desc) }
-  scope :with_includes, -> { includes(:account, status: STATUS_INCLUDES) }
-
-  delegate :target, :title, :content, :thread,
-           to: :status,
-           allow_nil: true
-
-  def object_type
-    orphaned? || targeted? ? :activity : status.object_type
-  end
-
-  def verb
-    orphaned? ? :delete : status.verb
-  end
-
-  def targeted?
-    [:follow, :request_friend, :authorize, :reject, :unfollow, :block, :unblock, :share, :favorite].include? verb
-  end
-
-  def threaded?
-    (verb == :favorite || object_type == :comment) && !thread.nil?
-  end
-
-  def mentions
-    orphaned? ? [] : status.active_mentions.map(&:account)
-  end
-
-  private
-
-  def orphaned?
-    status.nil?
-  end
-end
diff --git a/app/serializers/rest/account_serializer.rb b/app/serializers/rest/account_serializer.rb
index 12adc971c..272e3eb9c 100644
--- a/app/serializers/rest/account_serializer.rb
+++ b/app/serializers/rest/account_serializer.rb
@@ -29,7 +29,7 @@ class REST::AccountSerializer < ActiveModel::Serializer
   end
 
   def url
-    TagManager.instance.url_for(object)
+    ActivityPub::TagManager.instance.url_for(object)
   end
 
   def avatar
diff --git a/app/serializers/rest/status_serializer.rb b/app/serializers/rest/status_serializer.rb
index c9b76cb16..2dc4a1b61 100644
--- a/app/serializers/rest/status_serializer.rb
+++ b/app/serializers/rest/status_serializer.rb
@@ -58,7 +58,7 @@ class REST::StatusSerializer < ActiveModel::Serializer
   end
 
   def uri
-    OStatus::TagManager.instance.uri_for(object)
+    ActivityPub::TagManager.instance.uri_for(object)
   end
 
   def content
@@ -66,7 +66,7 @@ class REST::StatusSerializer < ActiveModel::Serializer
   end
 
   def url
-    TagManager.instance.url_for(object)
+    ActivityPub::TagManager.instance.url_for(object)
   end
 
   def favourited
@@ -132,7 +132,7 @@ class REST::StatusSerializer < ActiveModel::Serializer
     end
 
     def url
-      TagManager.instance.url_for(object.account)
+      ActivityPub::TagManager.instance.url_for(object.account)
     end
 
     def acct
diff --git a/app/serializers/rss/account_serializer.rb b/app/serializers/rss/account_serializer.rb
index 88eca79ed..278affe13 100644
--- a/app/serializers/rss/account_serializer.rb
+++ b/app/serializers/rss/account_serializer.rb
@@ -2,7 +2,7 @@
 
 class RSS::AccountSerializer
   include ActionView::Helpers::NumberHelper
-  include StreamEntriesHelper
+  include StatusesHelper
   include RoutingHelper
 
   def render(account, statuses)
@@ -10,7 +10,7 @@ class RSS::AccountSerializer
 
     builder.title("#{display_name(account)} (@#{account.local_username_and_domain})")
            .description(account_description(account))
-           .link(TagManager.instance.url_for(account))
+           .link(ActivityPub::TagManager.instance.url_for(account))
            .logo(full_pack_url('media/images/logo.svg'))
            .accent_color('2b90d9')
 
@@ -20,7 +20,7 @@ class RSS::AccountSerializer
     statuses.each do |status|
       builder.item do |item|
         item.title(status.title)
-            .link(TagManager.instance.url_for(status))
+            .link(ActivityPub::TagManager.instance.url_for(status))
             .pub_date(status.created_at)
             .description(status.spoiler_text.presence || Formatter.instance.format(status, inline_poll_options: true).to_str)
 
diff --git a/app/serializers/rss/tag_serializer.rb b/app/serializers/rss/tag_serializer.rb
index 644380149..e8562ee87 100644
--- a/app/serializers/rss/tag_serializer.rb
+++ b/app/serializers/rss/tag_serializer.rb
@@ -3,7 +3,7 @@
 class RSS::TagSerializer
   include ActionView::Helpers::NumberHelper
   include ActionView::Helpers::SanitizeHelper
-  include StreamEntriesHelper
+  include StatusesHelper
   include RoutingHelper
 
   def render(tag, statuses)
@@ -18,7 +18,7 @@ class RSS::TagSerializer
     statuses.each do |status|
       builder.item do |item|
         item.title(status.title)
-            .link(TagManager.instance.url_for(status))
+            .link(ActivityPub::TagManager.instance.url_for(status))
             .pub_date(status.created_at)
             .description(status.spoiler_text.presence || Formatter.instance.format(status).to_str)
 
diff --git a/app/services/batched_remove_status_service.rb b/app/services/batched_remove_status_service.rb
index cb66debc8..27dc460a6 100644
--- a/app/services/batched_remove_status_service.rb
+++ b/app/services/batched_remove_status_service.rb
@@ -12,7 +12,7 @@ class BatchedRemoveStatusService < BaseService
   # @param [Hash] options
   # @option [Boolean] :skip_side_effects
   def call(statuses, **options)
-    statuses = Status.where(id: statuses.map(&:id)).includes(:account, :stream_entry).flat_map { |status| [status] + status.reblogs.includes(:account, :stream_entry).to_a }
+    statuses = Status.where(id: statuses.map(&:id)).includes(:account).flat_map { |status| [status] + status.reblogs.includes(:account).to_a }
 
     @mentions = statuses.each_with_object({}) { |s, h| h[s.id] = s.active_mentions.includes(:account).to_a }
     @tags     = statuses.each_with_object({}) { |s, h| h[s.id] = s.tags.pluck(:name) }
diff --git a/app/services/fetch_link_card_service.rb b/app/services/fetch_link_card_service.rb
index 494aaed75..75fbd0e8c 100644
--- a/app/services/fetch_link_card_service.rb
+++ b/app/services/fetch_link_card_service.rb
@@ -84,7 +84,7 @@ class FetchLinkCardService < BaseService
 
   def mention_link?(a)
     @status.mentions.any? do |mention|
-      a['href'] == TagManager.instance.url_for(mention.account)
+      a['href'] == ActivityPub::TagManager.instance.url_for(mention.account)
     end
   end
 
diff --git a/app/services/process_mentions_service.rb b/app/services/process_mentions_service.rb
index da52bff6a..90dca9740 100644
--- a/app/services/process_mentions_service.rb
+++ b/app/services/process_mentions_service.rb
@@ -40,7 +40,7 @@ class ProcessMentionsService < BaseService
   private
 
   def mention_undeliverable?(mentioned_account)
-    mentioned_account.nil? || (!mentioned_account.local? && mentioned_account.ostatus? && @status.stream_entry.hidden?)
+    mentioned_account.nil? || (!mentioned_account.local? && mentioned_account.ostatus?)
   end
 
   def create_notification(mention)
diff --git a/app/services/remove_status_service.rb b/app/services/remove_status_service.rb
index a8c9100b3..6311971ff 100644
--- a/app/services/remove_status_service.rb
+++ b/app/services/remove_status_service.rb
@@ -5,14 +5,13 @@ class RemoveStatusService < BaseService
   include Payloadable
 
   def call(status, **options)
-    @payload      = Oj.dump(event: :delete, payload: status.id.to_s)
-    @status       = status
-    @account      = status.account
-    @tags         = status.tags.pluck(:name).to_a
-    @mentions     = status.active_mentions.includes(:account).to_a
-    @reblogs      = status.reblogs.includes(:account).to_a
-    @stream_entry = status.stream_entry
-    @options      = options
+    @payload  = Oj.dump(event: :delete, payload: status.id.to_s)
+    @status   = status
+    @account  = status.account
+    @tags     = status.tags.pluck(:name).to_a
+    @mentions = status.active_mentions.includes(:account).to_a
+    @reblogs  = status.reblogs.includes(:account).to_a
+    @options  = options
 
     RedisLock.acquire(lock_options) do |lock|
       if lock.acquired?
diff --git a/app/services/resolve_url_service.rb b/app/services/resolve_url_service.rb
index bbdc0a595..f941b489a 100644
--- a/app/services/resolve_url_service.rb
+++ b/app/services/resolve_url_service.rb
@@ -73,10 +73,7 @@ class ResolveURLService < BaseService
 
     return unless recognized_params[:action] == 'show'
 
-    if recognized_params[:controller] == 'stream_entries'
-      status = StreamEntry.find_by(id: recognized_params[:id])&.status
-      check_local_status(status)
-    elsif recognized_params[:controller] == 'statuses'
+    if recognized_params[:controller] == 'statuses'
       status = Status.find_by(id: recognized_params[:id])
       check_local_status(status)
     elsif recognized_params[:controller] == 'accounts'
diff --git a/app/services/suspend_account_service.rb b/app/services/suspend_account_service.rb
index a5ce3dbd9..0ebe0b562 100644
--- a/app/services/suspend_account_service.rb
+++ b/app/services/suspend_account_service.rb
@@ -24,7 +24,6 @@ class SuspendAccountService < BaseService
     report_notes
     scheduled_statuses
     status_pins
-    stream_entries
     subscriptions
   ).freeze
 
diff --git a/app/views/accounts/_moved.html.haml b/app/views/accounts/_moved.html.haml
index 7a777bfea..02fd7bf42 100644
--- a/app/views/accounts/_moved.html.haml
+++ b/app/views/accounts/_moved.html.haml
@@ -3,10 +3,10 @@
 .moved-account-widget
   .moved-account-widget__message
     = fa_icon 'suitcase'
-    = t('accounts.moved_html', name: content_tag(:bdi, content_tag(:strong, display_name(account, custom_emojify: true), class: :emojify)), new_profile_link: link_to(content_tag(:strong, safe_join(['@', content_tag(:span, moved_to_account.acct)])), TagManager.instance.url_for(moved_to_account), class: 'mention'))
+    = t('accounts.moved_html', name: content_tag(:bdi, content_tag(:strong, display_name(account, custom_emojify: true), class: :emojify)), new_profile_link: link_to(content_tag(:strong, safe_join(['@', content_tag(:span, moved_to_account.acct)])), ActivityPub::TagManager.instance.url_for(moved_to_account), class: 'mention'))
 
   .moved-account-widget__card
-    = link_to TagManager.instance.url_for(moved_to_account), class: 'detailed-status__display-name p-author h-card', target: '_blank', rel: 'me noopener' do
+    = link_to ActivityPub::TagManager.instance.url_for(moved_to_account), class: 'detailed-status__display-name p-author h-card', target: '_blank', rel: 'me noopener' do
       .detailed-status__display-avatar
         .account__avatar-overlay
           .account__avatar-overlay-base{ style: "background-image: url('#{moved_to_account.avatar.url(:original)}')" }
diff --git a/app/views/accounts/show.html.haml b/app/views/accounts/show.html.haml
index de7d2a8ba..0dc984dcc 100644
--- a/app/views/accounts/show.html.haml
+++ b/app/views/accounts/show.html.haml
@@ -39,12 +39,12 @@
       - else
         .activity-stream.activity-stream--under-tabs
           - if params[:page].to_i.zero?
-            = render partial: 'stream_entries/status', collection: @pinned_statuses, as: :status, locals: { pinned: true }
+            = render partial: 'statuses/status', collection: @pinned_statuses, as: :status, locals: { pinned: true }
 
           - if @newer_url
             .entry= link_to_more @newer_url
 
-          = render partial: 'stream_entries/status', collection: @statuses, as: :status
+          = render partial: 'statuses/status', collection: @statuses, as: :status
 
           - if @older_url
             .entry= link_to_more @older_url
diff --git a/app/views/admin/accounts/_account.html.haml b/app/views/admin/accounts/_account.html.haml
index eba3ad804..b057d3e42 100644
--- a/app/views/admin/accounts/_account.html.haml
+++ b/app/views/admin/accounts/_account.html.haml
@@ -19,4 +19,4 @@
       = table_link_to 'times', t('admin.accounts.reject'), reject_admin_account_path(account.id), method: :post, data: { confirm: t('admin.accounts.are_you_sure') } if can?(:reject, account.user)
     - else
       = table_link_to 'circle', t('admin.accounts.web'), web_path("accounts/#{account.id}")
-      = table_link_to 'globe', t('admin.accounts.public'), TagManager.instance.url_for(account)
+      = table_link_to 'globe', t('admin.accounts.public'), ActivityPub::TagManager.instance.url_for(account)
diff --git a/app/views/admin/reports/_status.html.haml b/app/views/admin/reports/_status.html.haml
index b3c145120..9376db7ff 100644
--- a/app/views/admin/reports/_status.html.haml
+++ b/app/views/admin/reports/_status.html.haml
@@ -19,7 +19,7 @@
         = react_component :media_gallery, height: 343, sensitive: !current_account&.user&.show_all_media? && status.sensitive? || current_account&.user&.hide_all_media?, 'autoPlayGif': current_account&.user&.setting_auto_play_gif, media: status.proper.media_attachments.map { |a| ActiveModelSerializers::SerializableResource.new(a, serializer: REST::MediaAttachmentSerializer).as_json }
 
     .detailed-status__meta
-      = link_to TagManager.instance.url_for(status), class: 'detailed-status__datetime', target: stream_link_target, rel: 'noopener' do
+      = link_to ActivityPub::TagManager.instance.url_for(status), class: 'detailed-status__datetime', target: stream_link_target, rel: 'noopener' do
         %time.formatted{ datetime: status.created_at.iso8601, title: l(status.created_at) }= l(status.created_at)
       ·
       - if status.reblog?
diff --git a/app/views/application/_card.html.haml b/app/views/application/_card.html.haml
index e6059b035..00254c40c 100644
--- a/app/views/application/_card.html.haml
+++ b/app/views/application/_card.html.haml
@@ -1,4 +1,4 @@
-- account_url = local_assigns[:admin] ? admin_account_path(account.id) : TagManager.instance.url_for(account)
+- account_url = local_assigns[:admin] ? admin_account_path(account.id) : ActivityPub::TagManager.instance.url_for(account)
 
 .card.h-card
   = link_to account_url, target: '_blank', rel: 'noopener' do
diff --git a/app/views/authorize_interactions/_post_follow_actions.html.haml b/app/views/authorize_interactions/_post_follow_actions.html.haml
index 561c60137..dd71160e2 100644
--- a/app/views/authorize_interactions/_post_follow_actions.html.haml
+++ b/app/views/authorize_interactions/_post_follow_actions.html.haml
@@ -1,4 +1,4 @@
 .post-follow-actions
   %div= link_to t('authorize_follow.post_follow.web'), web_url("accounts/#{@resource.id}"), class: 'button button--block'
-  %div= link_to t('authorize_follow.post_follow.return'), TagManager.instance.url_for(@resource), class: 'button button--block'
+  %div= link_to t('authorize_follow.post_follow.return'), ActivityPub::TagManager.instance.url_for(@resource), class: 'button button--block'
   %div= t('authorize_follow.post_follow.close')
diff --git a/app/views/remote_interaction/new.html.haml b/app/views/remote_interaction/new.html.haml
index c8c08991f..2cc0fcb93 100644
--- a/app/views/remote_interaction/new.html.haml
+++ b/app/views/remote_interaction/new.html.haml
@@ -7,7 +7,7 @@
 
     .public-layout
       .activity-stream.activity-stream--highlighted
-        = render 'stream_entries/status', status: @status
+        = render 'statuses/status', status: @status
 
   = simple_form_for @remote_follow, as: :remote_follow, url: remote_interaction_path(@status) do |f|
     = render 'shared/error_messages', object: @remote_follow
diff --git a/app/views/remote_unfollows/_card.html.haml b/app/views/remote_unfollows/_card.html.haml
index 9abcfd37e..80ad3bae2 100644
--- a/app/views/remote_unfollows/_card.html.haml
+++ b/app/views/remote_unfollows/_card.html.haml
@@ -4,7 +4,7 @@
       = image_tag account.avatar.url(:original), alt: '', width: 48, height: 48, class: 'avatar'
 
     %span.display-name
-      - account_url = local_assigns[:admin] ? admin_account_path(account.id) : TagManager.instance.url_for(account)
+      - account_url = local_assigns[:admin] ? admin_account_path(account.id) : ActivityPub::TagManager.instance.url_for(account)
       = link_to account_url, class: 'detailed-status__display-name p-author h-card', target: '_blank', rel: 'noopener' do
         %strong.emojify= display_name(account, custom_emojify: true)
         %span @#{account.acct}
diff --git a/app/views/remote_unfollows/_post_follow_actions.html.haml b/app/views/remote_unfollows/_post_follow_actions.html.haml
index 2a9c062e9..328f7c833 100644
--- a/app/views/remote_unfollows/_post_follow_actions.html.haml
+++ b/app/views/remote_unfollows/_post_follow_actions.html.haml
@@ -1,4 +1,4 @@
 .post-follow-actions
   %div= link_to t('authorize_follow.post_follow.web'), web_url("accounts/#{@account.id}"), class: 'button button--block'
-  %div= link_to t('authorize_follow.post_follow.return'), TagManager.instance.url_for(@account), class: 'button button--block'
+  %div= link_to t('authorize_follow.post_follow.return'), ActivityPub::TagManager.instance.url_for(@account), class: 'button button--block'
   %div= t('authorize_follow.post_follow.close')
diff --git a/app/views/statuses/_attachment_list.html.haml b/app/views/statuses/_attachment_list.html.haml
new file mode 100644
index 000000000..d9706f47b
--- /dev/null
+++ b/app/views/statuses/_attachment_list.html.haml
@@ -0,0 +1,8 @@
+.attachment-list
+  .attachment-list__icon
+    = fa_icon 'link'
+  %ul.attachment-list__list
+    - attachments.each do |media|
+      %li
+        - url = media.remote_url.presence || media.file.url
+        = link_to File.basename(url), url, title: media.description
diff --git a/app/views/statuses/_detailed_status.html.haml b/app/views/statuses/_detailed_status.html.haml
new file mode 100644
index 000000000..8686c2033
--- /dev/null
+++ b/app/views/statuses/_detailed_status.html.haml
@@ -0,0 +1,79 @@
+.detailed-status.detailed-status--flex
+  .p-author.h-card
+    = link_to ActivityPub::TagManager.instance.url_for(status.account), class: 'detailed-status__display-name u-url', target: stream_link_target, rel: 'noopener' do
+      .detailed-status__display-avatar
+        - if current_account&.user&.setting_auto_play_gif || autoplay
+          = image_tag status.account.avatar_original_url, width: 48, height: 48, alt: '', class: 'account__avatar u-photo'
+        - else
+          = image_tag status.account.avatar_static_url, width: 48, height: 48, alt: '', class: 'account__avatar u-photo'
+      %span.display-name
+        %bdi
+          %strong.display-name__html.p-name.emojify= display_name(status.account, custom_emojify: true, autoplay: autoplay)
+        %span.display-name__account
+          = acct(status.account)
+          = fa_icon('lock') if status.account.locked?
+
+  = account_action_button(status.account)
+
+  .status__content.emojify<
+    - if status.spoiler_text?
+      %p{ :style => ('margin-bottom: 0' unless current_account&.user&.setting_expand_spoilers) }<
+        %span.p-summary> #{Formatter.instance.format_spoiler(status, autoplay: autoplay)}&nbsp;
+        %button.status__content__spoiler-link= t('statuses.show_more')
+    .e-content{ lang: status.language, style: "display: #{!current_account&.user&.setting_expand_spoilers && status.spoiler_text? ? 'none' : 'block'}; direction: #{rtl_status?(status) ? 'rtl' : 'ltr'}" }
+      = Formatter.instance.format(status, custom_emojify: true, autoplay: autoplay)
+      - if status.preloadable_poll
+        = react_component :poll, disabled: true, poll: ActiveModelSerializers::SerializableResource.new(status.preloadable_poll, serializer: REST::PollSerializer, scope: current_user, scope_name: :current_user).as_json do
+          = render partial: 'statuses/poll', locals: { status: status, poll: status.preloadable_poll, autoplay: autoplay }
+
+  - if !status.media_attachments.empty?
+    - if status.media_attachments.first.audio_or_video?
+      - video = status.media_attachments.first
+      = react_component :video, src: video.file.url(:original), preview: video.file.url(:small), blurhash: video.blurhash, sensitive: !current_account&.user&.show_all_media? && status.sensitive? || current_account&.user&.hide_all_media?, width: 670, height: 380, detailed: true, inline: true, alt: video.description do
+        = render partial: 'statuses/attachment_list', locals: { attachments: status.media_attachments }
+    - else
+      = react_component :media_gallery, height: 380, sensitive: !current_account&.user&.show_all_media? && status.sensitive? || current_account&.user&.hide_all_media?, standalone: true, 'autoPlayGif': current_account&.user&.setting_auto_play_gif || autoplay, 'reduceMotion': current_account&.user&.setting_reduce_motion, media: status.media_attachments.map { |a| ActiveModelSerializers::SerializableResource.new(a, serializer: REST::MediaAttachmentSerializer).as_json } do
+        = render partial: 'statuses/attachment_list', locals: { attachments: status.media_attachments }
+  - elsif status.preview_card
+    = react_component :card, 'maxDescription': 160, card: ActiveModelSerializers::SerializableResource.new(status.preview_card, serializer: REST::PreviewCardSerializer).as_json
+
+  .detailed-status__meta
+    %data.dt-published{ value: status.created_at.to_time.iso8601 }
+
+    = link_to ActivityPub::TagManager.instance.url_for(status), class: 'detailed-status__datetime u-url u-uid', target: stream_link_target, rel: 'noopener' do
+      %time.formatted{ datetime: status.created_at.iso8601, title: l(status.created_at) }= l(status.created_at)
+    ·
+    - if status.application && @account.user&.setting_show_application
+      - if status.application.website.blank?
+        %strong.detailed-status__application= status.application.name
+      - else
+        = link_to status.application.name, status.application.website, class: 'detailed-status__application', target: '_blank', rel: 'noopener'
+      ·
+    = link_to remote_interaction_path(status, type: :reply), class: 'modal-button detailed-status__link' do
+      - if status.in_reply_to_id.nil?
+        = fa_icon('reply')
+      - else
+        = fa_icon('reply-all')
+      %span.detailed-status__reblogs>= number_to_human status.replies_count, strip_insignificant_zeros: true
+      = " "
+    ·
+    - if status.direct_visibility?
+      %span.detailed-status__link<
+        = fa_icon('envelope')
+    - elsif status.private_visibility? || status.limited_visibility?
+      %span.detailed-status__link<
+        = fa_icon('lock')
+    - else
+      = link_to remote_interaction_path(status, type: :reblog), class: 'modal-button detailed-status__link' do
+        = fa_icon('retweet')
+        %span.detailed-status__reblogs>= number_to_human status.reblogs_count, strip_insignificant_zeros: true
+        = " "
+    ·
+    = link_to remote_interaction_path(status, type: :favourite), class: 'modal-button detailed-status__link' do
+      = fa_icon('star')
+      %span.detailed-status__favorites>= number_to_human status.favourites_count, strip_insignificant_zeros: true
+      = " "
+
+    - if user_signed_in?
+      ·
+      = link_to t('statuses.open_in_web'), web_url("statuses/#{status.id}"), class: 'detailed-status__application', target: '_blank'
diff --git a/app/views/statuses/_og_description.html.haml b/app/views/statuses/_og_description.html.haml
new file mode 100644
index 000000000..a7b18424d
--- /dev/null
+++ b/app/views/statuses/_og_description.html.haml
@@ -0,0 +1,4 @@
+- description = status_description(activity)
+
+%meta{ name: 'description', content: description }/
+= opengraph 'og:description', description
diff --git a/app/views/statuses/_og_image.html.haml b/app/views/statuses/_og_image.html.haml
new file mode 100644
index 000000000..67f9274b6
--- /dev/null
+++ b/app/views/statuses/_og_image.html.haml
@@ -0,0 +1,38 @@
+- if activity.is_a?(Status) && (activity.non_sensitive_with_media? || (activity.with_media? && Setting.preview_sensitive_media))
+  - player_card = false
+  - activity.media_attachments.each do |media|
+    - if media.image?
+      = opengraph 'og:image', full_asset_url(media.file.url(:original))
+      = opengraph 'og:image:type', media.file_content_type
+      - unless media.file.meta.nil?
+        = opengraph 'og:image:width', media.file.meta.dig('original', 'width')
+        = opengraph 'og:image:height', media.file.meta.dig('original', 'height')
+      - if media.description.present?
+        = opengraph 'og:image:alt', media.description
+    - elsif media.video? || media.gifv?
+      - player_card = true
+      = opengraph 'og:image', full_asset_url(media.file.url(:small))
+      = opengraph 'og:image:type', 'image/png'
+      - unless media.file.meta.nil?
+        = opengraph 'og:image:width', media.file.meta.dig('small', 'width')
+        = opengraph 'og:image:height', media.file.meta.dig('small', 'height')
+      = opengraph 'og:video', full_asset_url(media.file.url(:original))
+      = opengraph 'og:video:secure_url', full_asset_url(media.file.url(:original))
+      = opengraph 'og:video:type', media.file_content_type
+      = opengraph 'twitter:player', medium_player_url(media)
+      = opengraph 'twitter:player:stream', full_asset_url(media.file.url(:original))
+      = opengraph 'twitter:player:stream:content_type', media.file_content_type
+      - unless media.file.meta.nil?
+        = opengraph 'og:video:width', media.file.meta.dig('original', 'width')
+        = opengraph 'og:video:height', media.file.meta.dig('original', 'height')
+        = opengraph 'twitter:player:width', media.file.meta.dig('original', 'width')
+        = opengraph 'twitter:player:height', media.file.meta.dig('original', 'height')
+  - if player_card
+    = opengraph 'twitter:card', 'player'
+  - else
+    = opengraph 'twitter:card', 'summary_large_image'
+- else
+  = opengraph 'og:image', full_asset_url(account.avatar.url(:original))
+  = opengraph 'og:image:width', '120'
+  = opengraph 'og:image:height','120'
+  = opengraph 'twitter:card', 'summary'
diff --git a/app/views/statuses/_poll.html.haml b/app/views/statuses/_poll.html.haml
new file mode 100644
index 000000000..ba34890df
--- /dev/null
+++ b/app/views/statuses/_poll.html.haml
@@ -0,0 +1,27 @@
+- show_results = (user_signed_in? && poll.voted?(current_account)) || poll.expired?
+
+.poll
+  %ul
+    - poll.loaded_options.each do |option|
+      %li
+        - if show_results
+          - percent = poll.votes_count > 0 ? 100 * option.votes_count / poll.votes_count : 0
+          %span.poll__chart{ style: "width: #{percent}%" }
+
+          %label.poll__text><
+            %span.poll__number= percent.round
+            = Formatter.instance.format_poll_option(status, option, autoplay: autoplay)
+        - else
+          %label.poll__text><
+            %span.poll__input{ class: poll.multiple? ? 'checkbox' : nil}><
+            = Formatter.instance.format_poll_option(status, option, autoplay: autoplay)
+  .poll__footer
+    - unless show_results
+      %button.button.button-secondary{ disabled: true }
+        = t('statuses.poll.vote')
+
+    %span= t('statuses.poll.total_votes', count: poll.votes_count)
+
+    - unless poll.expires_at.nil?
+      ·
+      %span= l poll.expires_at
diff --git a/app/views/statuses/_simple_status.html.haml b/app/views/statuses/_simple_status.html.haml
new file mode 100644
index 000000000..11220dfcb
--- /dev/null
+++ b/app/views/statuses/_simple_status.html.haml
@@ -0,0 +1,60 @@
+.status
+  .status__info
+    = link_to ActivityPub::TagManager.instance.url_for(status), class: 'status__relative-time u-url u-uid', target: stream_link_target, rel: 'noopener' do
+      %time.time-ago{ datetime: status.created_at.iso8601, title: l(status.created_at) }= l(status.created_at)
+    %data.dt-published{ value: status.created_at.to_time.iso8601 }
+
+    .p-author.h-card
+      = link_to ActivityPub::TagManager.instance.url_for(status.account), class: 'status__display-name u-url', target: stream_link_target, rel: 'noopener' do
+        .status__avatar
+          %div
+            - if current_account&.user&.setting_auto_play_gif || autoplay
+              = image_tag status.account.avatar_original_url, width: 48, height: 48, alt: '', class: 'u-photo account__avatar'
+            - else
+              = image_tag status.account.avatar_static_url, width: 48, height: 48, alt: '', class: 'u-photo account__avatar'
+        %span.display-name
+          %bdi
+            %strong.display-name__html.p-name.emojify= display_name(status.account, custom_emojify: true, autoplay: autoplay)
+          &nbsp;
+          %span.display-name__account
+            = acct(status.account)
+            = fa_icon('lock') if status.account.locked?
+  .status__content.emojify<
+    - if status.spoiler_text?
+      %p{ :style => ('margin-bottom: 0' unless current_account&.user&.setting_expand_spoilers) }<
+        %span.p-summary> #{Formatter.instance.format_spoiler(status, autoplay: autoplay)}&nbsp;
+        %button.status__content__spoiler-link= t('statuses.show_more')
+    .e-content{ lang: status.language, style: "display: #{!current_account&.user&.setting_expand_spoilers && status.spoiler_text? ? 'none' : 'block'}; direction: #{rtl_status?(status) ? 'rtl' : 'ltr'}" }
+      = Formatter.instance.format(status, custom_emojify: true, autoplay: autoplay)
+      - if status.preloadable_poll
+        = react_component :poll, disabled: true, poll: ActiveModelSerializers::SerializableResource.new(status.preloadable_poll, serializer: REST::PollSerializer, scope: current_user, scope_name: :current_user).as_json do
+          = render partial: 'statuses/poll', locals: { status: status, poll: status.preloadable_poll, autoplay: autoplay }
+
+  - if !status.media_attachments.empty?
+    - if status.media_attachments.first.audio_or_video?
+      - video = status.media_attachments.first
+      = react_component :video, src: video.file.url(:original), preview: video.file.url(:small), blurhash: video.blurhash, sensitive: !current_account&.user&.show_all_media? && status.sensitive? || current_account&.user&.hide_all_media?, width: 610, height: 343, inline: true, alt: video.description do
+        = render partial: 'statuses/attachment_list', locals: { attachments: status.media_attachments }
+    - else
+      = react_component :media_gallery, height: 343, sensitive: !current_account&.user&.show_all_media? && status.sensitive? || current_account&.user&.hide_all_media?, 'autoPlayGif': current_account&.user&.setting_auto_play_gif || autoplay, media: status.media_attachments.map { |a| ActiveModelSerializers::SerializableResource.new(a, serializer: REST::MediaAttachmentSerializer).as_json } do
+        = render partial: 'statuses/attachment_list', locals: { attachments: status.media_attachments }
+  - elsif status.preview_card
+    = react_component :card, 'maxDescription': 160, card: ActiveModelSerializers::SerializableResource.new(status.preview_card, serializer: REST::PreviewCardSerializer).as_json
+
+  .status__action-bar
+    .status__action-bar__counter
+      = link_to remote_interaction_path(status, type: :reply), class: 'status__action-bar-button icon-button modal-button', style: 'font-size: 18px; width: 23.1429px; height: 23.1429px; line-height: 23.15px;' do
+        - if status.in_reply_to_id.nil?
+          = fa_icon 'reply fw'
+        - else
+          = fa_icon 'reply-all fw'
+      .status__action-bar__counter__label= obscured_counter status.replies_count
+    = link_to remote_interaction_path(status, type: :reblog), class: 'status__action-bar-button icon-button modal-button', style: 'font-size: 18px; width: 23.1429px; height: 23.1429px; line-height: 23.15px;' do
+      - if status.public_visibility? || status.unlisted_visibility?
+        = fa_icon 'retweet fw'
+      - elsif status.private_visibility?
+        = fa_icon 'lock fw'
+      - else
+        = fa_icon 'envelope fw'
+    = link_to remote_interaction_path(status, type: :favourite), class: 'status__action-bar-button icon-button modal-button', style: 'font-size: 18px; width: 23.1429px; height: 23.1429px; line-height: 23.15px;' do
+      = fa_icon 'star fw'
diff --git a/app/views/statuses/_status.html.haml b/app/views/statuses/_status.html.haml
new file mode 100644
index 000000000..0e3652503
--- /dev/null
+++ b/app/views/statuses/_status.html.haml
@@ -0,0 +1,62 @@
+:ruby
+  pinned          ||= false
+  include_threads ||= false
+  is_predecessor  ||= false
+  is_successor    ||= false
+  direct_reply_id ||= false
+  parent_id       ||= false
+  autoplay        ||= current_account&.user&.setting_auto_play_gif
+  is_direct_parent  = direct_reply_id == status.id
+  is_direct_child   = parent_id == status.in_reply_to_id
+  centered        ||= include_threads && !is_predecessor && !is_successor
+  h_class           = microformats_h_class(status, is_predecessor, is_successor, include_threads)
+  style_classes     = style_classes(status, is_predecessor, is_successor, include_threads)
+  mf_classes        = microformats_classes(status, is_direct_parent, is_direct_child)
+  entry_classes     = h_class + ' ' + mf_classes + ' ' + style_classes
+
+- if status.reply? && include_threads
+  - if @next_ancestor
+    .entry{ class: entry_classes }
+      = link_to_more ActivityPub::TagManager.instance.url_for(@next_ancestor)
+
+  = render partial: 'statuses/status', collection: @ancestors, as: :status, locals: { is_predecessor: true, direct_reply_id: status.in_reply_to_id }, autoplay: autoplay
+
+.entry{ class: entry_classes }
+
+  - if status.reblog?
+    .status__prepend
+      .status__prepend-icon-wrapper
+        %i.status__prepend-icon.fa.fa-fw.fa-retweet
+      %span
+        = link_to ActivityPub::TagManager.instance.url_for(status.account), class: 'status__display-name muted' do
+          %bdi
+            %strong.emojify= display_name(status.account, custom_emojify: true)
+        = t('stream_entries.reblogged')
+  - elsif pinned
+    .status__prepend
+      .status__prepend-icon-wrapper
+        %i.status__prepend-icon.fa.fa-fw.fa-thumb-tack
+      %span
+        = t('stream_entries.pinned')
+
+  = render (centered ? 'statuses/detailed_status' : 'statuses/simple_status'), status: status.proper, autoplay: autoplay
+
+- if include_threads
+  - if @since_descendant_thread_id
+    .entry{ class: entry_classes }
+      = link_to_more short_account_status_url(status.account.username, status, max_descendant_thread_id: @since_descendant_thread_id + 1)
+  - @descendant_threads.each do |thread|
+    = render partial: 'statuses/status', collection: thread[:statuses], as: :status, locals: { is_successor: true, parent_id: status.id }, autoplay: autoplay
+
+    - if thread[:next_status]
+      .entry{ class: entry_classes }
+        = link_to_more ActivityPub::TagManager.instance.url_for(thread[:next_status])
+  - if @next_descendant_thread
+    .entry{ class: entry_classes }
+      = link_to_more short_account_status_url(status.account.username, status, since_descendant_thread_id: @max_descendant_thread_id - 1)
+
+- if include_threads && !embedded_view? && !user_signed_in?
+  .entry{ class: entry_classes }
+    = link_to new_user_session_path, class: 'load-more load-gap' do
+      = fa_icon 'comments'
+      = t('statuses.sign_in_to_participate')
diff --git a/app/views/statuses/embed.html.haml b/app/views/statuses/embed.html.haml
new file mode 100644
index 000000000..6f2ec646f
--- /dev/null
+++ b/app/views/statuses/embed.html.haml
@@ -0,0 +1,3 @@
+- cache @status do
+  .activity-stream.activity-stream--headless
+    = render 'status', status: @status, centered: true, autoplay: @autoplay
diff --git a/app/views/statuses/show.html.haml b/app/views/statuses/show.html.haml
new file mode 100644
index 000000000..704e37a3d
--- /dev/null
+++ b/app/views/statuses/show.html.haml
@@ -0,0 +1,24 @@
+- content_for :page_title do
+  = t('statuses.title', name: display_name(@account), quote: truncate(@status.spoiler_text.presence || @status.text, length: 50, omission: '…', escape: false))
+
+- content_for :header_tags do
+  - if @account.user&.setting_noindex
+    %meta{ name: 'robots', content: 'noindex' }/
+
+  %link{ rel: 'alternate', type: 'application/json+oembed', href: api_oembed_url(url: short_account_status_url(@account, @status), format: 'json') }/
+  %link{ rel: 'alternate', type: 'application/activity+json', href: ActivityPub::TagManager.instance.uri_for(@status) }/
+
+  = opengraph 'og:site_name', site_title
+  = opengraph 'og:type', 'article'
+  = opengraph 'og:title', "#{display_name(@account)} (@#{@account.local_username_and_domain})"
+  = opengraph 'og:url', short_account_status_url(@account, @status)
+
+  = render 'og_description', activity: @status
+  = render 'og_image', activity: @status, account: @account
+
+.grid
+  .column-0
+    .activity-stream.h-entry
+      = render partial: 'status', locals: { status: @status, include_threads: true }
+  .column-1
+    = render 'application/sidebar'
diff --git a/app/views/stream_entries/_attachment_list.html.haml b/app/views/stream_entries/_attachment_list.html.haml
deleted file mode 100644
index d9706f47b..000000000
--- a/app/views/stream_entries/_attachment_list.html.haml
+++ /dev/null
@@ -1,8 +0,0 @@
-.attachment-list
-  .attachment-list__icon
-    = fa_icon 'link'
-  %ul.attachment-list__list
-    - attachments.each do |media|
-      %li
-        - url = media.remote_url.presence || media.file.url
-        = link_to File.basename(url), url, title: media.description
diff --git a/app/views/stream_entries/_detailed_status.html.haml b/app/views/stream_entries/_detailed_status.html.haml
deleted file mode 100644
index 069d0053f..000000000
--- a/app/views/stream_entries/_detailed_status.html.haml
+++ /dev/null
@@ -1,79 +0,0 @@
-.detailed-status.detailed-status--flex
-  .p-author.h-card
-    = link_to TagManager.instance.url_for(status.account), class: 'detailed-status__display-name u-url', target: stream_link_target, rel: 'noopener' do
-      .detailed-status__display-avatar
-        - if current_account&.user&.setting_auto_play_gif || autoplay
-          = image_tag status.account.avatar_original_url, width: 48, height: 48, alt: '', class: 'account__avatar u-photo'
-        - else
-          = image_tag status.account.avatar_static_url, width: 48, height: 48, alt: '', class: 'account__avatar u-photo'
-      %span.display-name
-        %bdi
-          %strong.display-name__html.p-name.emojify= display_name(status.account, custom_emojify: true, autoplay: autoplay)
-        %span.display-name__account
-          = acct(status.account)
-          = fa_icon('lock') if status.account.locked?
-
-  = account_action_button(status.account)
-
-  .status__content.emojify<
-    - if status.spoiler_text?
-      %p{ :style => ('margin-bottom: 0' unless current_account&.user&.setting_expand_spoilers) }<
-        %span.p-summary> #{Formatter.instance.format_spoiler(status, autoplay: autoplay)}&nbsp;
-        %button.status__content__spoiler-link= t('statuses.show_more')
-    .e-content{ lang: status.language, style: "display: #{!current_account&.user&.setting_expand_spoilers && status.spoiler_text? ? 'none' : 'block'}; direction: #{rtl_status?(status) ? 'rtl' : 'ltr'}" }
-      = Formatter.instance.format(status, custom_emojify: true, autoplay: autoplay)
-      - if status.preloadable_poll
-        = react_component :poll, disabled: true, poll: ActiveModelSerializers::SerializableResource.new(status.preloadable_poll, serializer: REST::PollSerializer, scope: current_user, scope_name: :current_user).as_json do
-          = render partial: 'stream_entries/poll', locals: { status: status, poll: status.preloadable_poll, autoplay: autoplay }
-
-  - if !status.media_attachments.empty?
-    - if status.media_attachments.first.audio_or_video?
-      - video = status.media_attachments.first
-      = react_component :video, src: video.file.url(:original), preview: video.file.url(:small), blurhash: video.blurhash, sensitive: !current_account&.user&.show_all_media? && status.sensitive? || current_account&.user&.hide_all_media?, width: 670, height: 380, detailed: true, inline: true, alt: video.description do
-        = render partial: 'stream_entries/attachment_list', locals: { attachments: status.media_attachments }
-    - else
-      = react_component :media_gallery, height: 380, sensitive: !current_account&.user&.show_all_media? && status.sensitive? || current_account&.user&.hide_all_media?, standalone: true, 'autoPlayGif': current_account&.user&.setting_auto_play_gif || autoplay, 'reduceMotion': current_account&.user&.setting_reduce_motion, media: status.media_attachments.map { |a| ActiveModelSerializers::SerializableResource.new(a, serializer: REST::MediaAttachmentSerializer).as_json } do
-        = render partial: 'stream_entries/attachment_list', locals: { attachments: status.media_attachments }
-  - elsif status.preview_card
-    = react_component :card, 'maxDescription': 160, card: ActiveModelSerializers::SerializableResource.new(status.preview_card, serializer: REST::PreviewCardSerializer).as_json
-
-  .detailed-status__meta
-    %data.dt-published{ value: status.created_at.to_time.iso8601 }
-
-    = link_to TagManager.instance.url_for(status), class: 'detailed-status__datetime u-url u-uid', target: stream_link_target, rel: 'noopener' do
-      %time.formatted{ datetime: status.created_at.iso8601, title: l(status.created_at) }= l(status.created_at)
-    ·
-    - if status.application && @account.user&.setting_show_application
-      - if status.application.website.blank?
-        %strong.detailed-status__application= status.application.name
-      - else
-        = link_to status.application.name, status.application.website, class: 'detailed-status__application', target: '_blank', rel: 'noopener'
-      ·
-    = link_to remote_interaction_path(status, type: :reply), class: 'modal-button detailed-status__link' do
-      - if status.in_reply_to_id.nil?
-        = fa_icon('reply')
-      - else
-        = fa_icon('reply-all')
-      %span.detailed-status__reblogs>= number_to_human status.replies_count, strip_insignificant_zeros: true
-      = " "
-    ·
-    - if status.direct_visibility?
-      %span.detailed-status__link<
-        = fa_icon('envelope')
-    - elsif status.private_visibility? || status.limited_visibility?
-      %span.detailed-status__link<
-        = fa_icon('lock')
-    - else
-      = link_to remote_interaction_path(status, type: :reblog), class: 'modal-button detailed-status__link' do
-        = fa_icon('retweet')
-        %span.detailed-status__reblogs>= number_to_human status.reblogs_count, strip_insignificant_zeros: true
-        = " "
-    ·
-    = link_to remote_interaction_path(status, type: :favourite), class: 'modal-button detailed-status__link' do
-      = fa_icon('star')
-      %span.detailed-status__favorites>= number_to_human status.favourites_count, strip_insignificant_zeros: true
-      = " "
-
-    - if user_signed_in?
-      ·
-      = link_to t('statuses.open_in_web'), web_url("statuses/#{status.id}"), class: 'detailed-status__application', target: '_blank'
diff --git a/app/views/stream_entries/_og_description.html.haml b/app/views/stream_entries/_og_description.html.haml
deleted file mode 100644
index a7b18424d..000000000
--- a/app/views/stream_entries/_og_description.html.haml
+++ /dev/null
@@ -1,4 +0,0 @@
-- description = status_description(activity)
-
-%meta{ name: 'description', content: description }/
-= opengraph 'og:description', description
diff --git a/app/views/stream_entries/_og_image.html.haml b/app/views/stream_entries/_og_image.html.haml
deleted file mode 100644
index 67f9274b6..000000000
--- a/app/views/stream_entries/_og_image.html.haml
+++ /dev/null
@@ -1,38 +0,0 @@
-- if activity.is_a?(Status) && (activity.non_sensitive_with_media? || (activity.with_media? && Setting.preview_sensitive_media))
-  - player_card = false
-  - activity.media_attachments.each do |media|
-    - if media.image?
-      = opengraph 'og:image', full_asset_url(media.file.url(:original))
-      = opengraph 'og:image:type', media.file_content_type
-      - unless media.file.meta.nil?
-        = opengraph 'og:image:width', media.file.meta.dig('original', 'width')
-        = opengraph 'og:image:height', media.file.meta.dig('original', 'height')
-      - if media.description.present?
-        = opengraph 'og:image:alt', media.description
-    - elsif media.video? || media.gifv?
-      - player_card = true
-      = opengraph 'og:image', full_asset_url(media.file.url(:small))
-      = opengraph 'og:image:type', 'image/png'
-      - unless media.file.meta.nil?
-        = opengraph 'og:image:width', media.file.meta.dig('small', 'width')
-        = opengraph 'og:image:height', media.file.meta.dig('small', 'height')
-      = opengraph 'og:video', full_asset_url(media.file.url(:original))
-      = opengraph 'og:video:secure_url', full_asset_url(media.file.url(:original))
-      = opengraph 'og:video:type', media.file_content_type
-      = opengraph 'twitter:player', medium_player_url(media)
-      = opengraph 'twitter:player:stream', full_asset_url(media.file.url(:original))
-      = opengraph 'twitter:player:stream:content_type', media.file_content_type
-      - unless media.file.meta.nil?
-        = opengraph 'og:video:width', media.file.meta.dig('original', 'width')
-        = opengraph 'og:video:height', media.file.meta.dig('original', 'height')
-        = opengraph 'twitter:player:width', media.file.meta.dig('original', 'width')
-        = opengraph 'twitter:player:height', media.file.meta.dig('original', 'height')
-  - if player_card
-    = opengraph 'twitter:card', 'player'
-  - else
-    = opengraph 'twitter:card', 'summary_large_image'
-- else
-  = opengraph 'og:image', full_asset_url(account.avatar.url(:original))
-  = opengraph 'og:image:width', '120'
-  = opengraph 'og:image:height','120'
-  = opengraph 'twitter:card', 'summary'
diff --git a/app/views/stream_entries/_poll.html.haml b/app/views/stream_entries/_poll.html.haml
deleted file mode 100644
index ba34890df..000000000
--- a/app/views/stream_entries/_poll.html.haml
+++ /dev/null
@@ -1,27 +0,0 @@
-- show_results = (user_signed_in? && poll.voted?(current_account)) || poll.expired?
-
-.poll
-  %ul
-    - poll.loaded_options.each do |option|
-      %li
-        - if show_results
-          - percent = poll.votes_count > 0 ? 100 * option.votes_count / poll.votes_count : 0
-          %span.poll__chart{ style: "width: #{percent}%" }
-
-          %label.poll__text><
-            %span.poll__number= percent.round
-            = Formatter.instance.format_poll_option(status, option, autoplay: autoplay)
-        - else
-          %label.poll__text><
-            %span.poll__input{ class: poll.multiple? ? 'checkbox' : nil}><
-            = Formatter.instance.format_poll_option(status, option, autoplay: autoplay)
-  .poll__footer
-    - unless show_results
-      %button.button.button-secondary{ disabled: true }
-        = t('statuses.poll.vote')
-
-    %span= t('statuses.poll.total_votes', count: poll.votes_count)
-
-    - unless poll.expires_at.nil?
-      ·
-      %span= l poll.expires_at
diff --git a/app/views/stream_entries/_simple_status.html.haml b/app/views/stream_entries/_simple_status.html.haml
deleted file mode 100644
index 0b924f72f..000000000
--- a/app/views/stream_entries/_simple_status.html.haml
+++ /dev/null
@@ -1,60 +0,0 @@
-.status
-  .status__info
-    = link_to TagManager.instance.url_for(status), class: 'status__relative-time u-url u-uid', target: stream_link_target, rel: 'noopener' do
-      %time.time-ago{ datetime: status.created_at.iso8601, title: l(status.created_at) }= l(status.created_at)
-    %data.dt-published{ value: status.created_at.to_time.iso8601 }
-
-    .p-author.h-card
-      = link_to TagManager.instance.url_for(status.account), class: 'status__display-name u-url', target: stream_link_target, rel: 'noopener' do
-        .status__avatar
-          %div
-            - if current_account&.user&.setting_auto_play_gif || autoplay
-              = image_tag status.account.avatar_original_url, width: 48, height: 48, alt: '', class: 'u-photo account__avatar'
-            - else
-              = image_tag status.account.avatar_static_url, width: 48, height: 48, alt: '', class: 'u-photo account__avatar'
-        %span.display-name
-          %bdi
-            %strong.display-name__html.p-name.emojify= display_name(status.account, custom_emojify: true, autoplay: autoplay)
-          &nbsp;
-          %span.display-name__account
-            = acct(status.account)
-            = fa_icon('lock') if status.account.locked?
-  .status__content.emojify<
-    - if status.spoiler_text?
-      %p{ :style => ('margin-bottom: 0' unless current_account&.user&.setting_expand_spoilers) }<
-        %span.p-summary> #{Formatter.instance.format_spoiler(status, autoplay: autoplay)}&nbsp;
-        %button.status__content__spoiler-link= t('statuses.show_more')
-    .e-content{ lang: status.language, style: "display: #{!current_account&.user&.setting_expand_spoilers && status.spoiler_text? ? 'none' : 'block'}; direction: #{rtl_status?(status) ? 'rtl' : 'ltr'}" }
-      = Formatter.instance.format(status, custom_emojify: true, autoplay: autoplay)
-      - if status.preloadable_poll
-        = react_component :poll, disabled: true, poll: ActiveModelSerializers::SerializableResource.new(status.preloadable_poll, serializer: REST::PollSerializer, scope: current_user, scope_name: :current_user).as_json do
-          = render partial: 'stream_entries/poll', locals: { status: status, poll: status.preloadable_poll, autoplay: autoplay }
-
-  - if !status.media_attachments.empty?
-    - if status.media_attachments.first.audio_or_video?
-      - video = status.media_attachments.first
-      = react_component :video, src: video.file.url(:original), preview: video.file.url(:small), blurhash: video.blurhash, sensitive: !current_account&.user&.show_all_media? && status.sensitive? || current_account&.user&.hide_all_media?, width: 610, height: 343, inline: true, alt: video.description do
-        = render partial: 'stream_entries/attachment_list', locals: { attachments: status.media_attachments }
-    - else
-      = react_component :media_gallery, height: 343, sensitive: !current_account&.user&.show_all_media? && status.sensitive? || current_account&.user&.hide_all_media?, 'autoPlayGif': current_account&.user&.setting_auto_play_gif || autoplay, media: status.media_attachments.map { |a| ActiveModelSerializers::SerializableResource.new(a, serializer: REST::MediaAttachmentSerializer).as_json } do
-        = render partial: 'stream_entries/attachment_list', locals: { attachments: status.media_attachments }
-  - elsif status.preview_card
-    = react_component :card, 'maxDescription': 160, card: ActiveModelSerializers::SerializableResource.new(status.preview_card, serializer: REST::PreviewCardSerializer).as_json
-
-  .status__action-bar
-    .status__action-bar__counter
-      = link_to remote_interaction_path(status, type: :reply), class: 'status__action-bar-button icon-button modal-button', style: 'font-size: 18px; width: 23.1429px; height: 23.1429px; line-height: 23.15px;' do
-        - if status.in_reply_to_id.nil?
-          = fa_icon 'reply fw'
-        - else
-          = fa_icon 'reply-all fw'
-      .status__action-bar__counter__label= obscured_counter status.replies_count
-    = link_to remote_interaction_path(status, type: :reblog), class: 'status__action-bar-button icon-button modal-button', style: 'font-size: 18px; width: 23.1429px; height: 23.1429px; line-height: 23.15px;' do
-      - if status.public_visibility? || status.unlisted_visibility?
-        = fa_icon 'retweet fw'
-      - elsif status.private_visibility?
-        = fa_icon 'lock fw'
-      - else
-        = fa_icon 'envelope fw'
-    = link_to remote_interaction_path(status, type: :favourite), class: 'status__action-bar-button icon-button modal-button', style: 'font-size: 18px; width: 23.1429px; height: 23.1429px; line-height: 23.15px;' do
-      = fa_icon 'star fw'
diff --git a/app/views/stream_entries/_status.html.haml b/app/views/stream_entries/_status.html.haml
deleted file mode 100644
index 83887cd87..000000000
--- a/app/views/stream_entries/_status.html.haml
+++ /dev/null
@@ -1,62 +0,0 @@
-:ruby
-  pinned          ||= false
-  include_threads ||= false
-  is_predecessor  ||= false
-  is_successor    ||= false
-  direct_reply_id ||= false
-  parent_id       ||= false
-  autoplay        ||= current_account&.user&.setting_auto_play_gif
-  is_direct_parent  = direct_reply_id == status.id
-  is_direct_child   = parent_id == status.in_reply_to_id
-  centered        ||= include_threads && !is_predecessor && !is_successor
-  h_class           = microformats_h_class(status, is_predecessor, is_successor, include_threads)
-  style_classes     = style_classes(status, is_predecessor, is_successor, include_threads)
-  mf_classes        = microformats_classes(status, is_direct_parent, is_direct_child)
-  entry_classes     = h_class + ' ' + mf_classes + ' ' + style_classes
-
-- if status.reply? && include_threads
-  - if @next_ancestor
-    .entry{ class: entry_classes }
-      = link_to_more TagManager.instance.url_for(@next_ancestor)
-
-  = render partial: 'stream_entries/status', collection: @ancestors, as: :status, locals: { is_predecessor: true, direct_reply_id: status.in_reply_to_id }, autoplay: autoplay
-
-.entry{ class: entry_classes }
-
-  - if status.reblog?
-    .status__prepend
-      .status__prepend-icon-wrapper
-        %i.status__prepend-icon.fa.fa-fw.fa-retweet
-      %span
-        = link_to TagManager.instance.url_for(status.account), class: 'status__display-name muted' do
-          %bdi
-            %strong.emojify= display_name(status.account, custom_emojify: true)
-        = t('stream_entries.reblogged')
-  - elsif pinned
-    .status__prepend
-      .status__prepend-icon-wrapper
-        %i.status__prepend-icon.fa.fa-fw.fa-thumb-tack
-      %span
-        = t('stream_entries.pinned')
-
-  = render (centered ? 'stream_entries/detailed_status' : 'stream_entries/simple_status'), status: status.proper, autoplay: autoplay
-
-- if include_threads
-  - if @since_descendant_thread_id
-    .entry{ class: entry_classes }
-      = link_to_more short_account_status_url(status.account.username, status, max_descendant_thread_id: @since_descendant_thread_id + 1)
-  - @descendant_threads.each do |thread|
-    = render partial: 'stream_entries/status', collection: thread[:statuses], as: :status, locals: { is_successor: true, parent_id: status.id }, autoplay: autoplay
-
-    - if thread[:next_status]
-      .entry{ class: entry_classes }
-        = link_to_more TagManager.instance.url_for(thread[:next_status])
-  - if @next_descendant_thread
-    .entry{ class: entry_classes }
-      = link_to_more short_account_status_url(status.account.username, status, since_descendant_thread_id: @max_descendant_thread_id - 1)
-
-- if include_threads && !embedded_view? && !user_signed_in?
-  .entry{ class: entry_classes }
-    = link_to new_user_session_path, class: 'load-more load-gap' do
-      = fa_icon 'comments'
-      = t('statuses.sign_in_to_participate')
diff --git a/app/views/stream_entries/embed.html.haml b/app/views/stream_entries/embed.html.haml
deleted file mode 100644
index 4871c101e..000000000
--- a/app/views/stream_entries/embed.html.haml
+++ /dev/null
@@ -1,3 +0,0 @@
-- cache @stream_entry.activity do
-  .activity-stream.activity-stream--headless
-    = render "stream_entries/#{@type}", @type.to_sym => @stream_entry.activity, centered: true, autoplay: @autoplay
diff --git a/app/views/stream_entries/show.html.haml b/app/views/stream_entries/show.html.haml
deleted file mode 100644
index 0e81c4f68..000000000
--- a/app/views/stream_entries/show.html.haml
+++ /dev/null
@@ -1,25 +0,0 @@
-- content_for :page_title do
-  = t('statuses.title', name: display_name(@account), quote: truncate(@stream_entry.activity.spoiler_text.presence || @stream_entry.activity.text, length: 50, omission: '…', escape: false))
-
-- content_for :header_tags do
-  - if @account.user&.setting_noindex
-    %meta{ name: 'robots', content: 'noindex' }/
-
-  %link{ rel: 'alternate', type: 'application/atom+xml', href: account_stream_entry_url(@account, @stream_entry, format: 'atom') }/
-  %link{ rel: 'alternate', type: 'application/json+oembed', href: api_oembed_url(url: account_stream_entry_url(@account, @stream_entry), format: 'json') }/
-  %link{ rel: 'alternate', type: 'application/activity+json', href: ActivityPub::TagManager.instance.uri_for(@stream_entry.activity) }/
-
-  = opengraph 'og:site_name', site_title
-  = opengraph 'og:type', 'article'
-  = opengraph 'og:title', "#{display_name(@account)} (@#{@account.local_username_and_domain})"
-  = opengraph 'og:url', short_account_status_url(@account, @stream_entry.activity)
-
-  = render 'stream_entries/og_description', activity: @stream_entry.activity
-  = render 'stream_entries/og_image', activity: @stream_entry.activity, account: @account
-
-.grid
-  .column-0
-    .activity-stream.h-entry
-      = render partial: "stream_entries/#{@type}", locals: { @type.to_sym => @stream_entry.activity, include_threads: true }
-  .column-1
-    = render 'application/sidebar'
diff --git a/config/routes.rb b/config/routes.rb
index 4b6d464c6..69b495a96 100644
--- a/config/routes.rb
+++ b/config/routes.rb
@@ -45,12 +45,6 @@ Rails.application.routes.draw do
   get '/authorize_follow', to: redirect { |_, request| "/authorize_interaction?#{request.params.to_query}" }
 
   resources :accounts, path: 'users', only: [:show], param: :username do
-    resources :stream_entries, path: 'updates', only: [:show] do
-      member do
-        get :embed
-      end
-    end
-
     get :remote_follow,  to: 'remote_follow#new'
     post :remote_follow, to: 'remote_follow#create'
 
diff --git a/db/post_migrate/20190706233204_drop_stream_entries.rb b/db/post_migrate/20190706233204_drop_stream_entries.rb
new file mode 100644
index 000000000..1fecece05
--- /dev/null
+++ b/db/post_migrate/20190706233204_drop_stream_entries.rb
@@ -0,0 +1,13 @@
+# frozen_string_literal: true
+
+class DropStreamEntries < ActiveRecord::Migration[5.2]
+  disable_ddl_transaction!
+
+  def up
+    drop_table :stream_entries
+  end
+
+  def down
+    raise ActiveRecord::IrreversibleMigration
+  end
+end
diff --git a/db/schema.rb b/db/schema.rb
index 09e6c9fae..2e38fb1f2 100644
--- a/db/schema.rb
+++ b/db/schema.rb
@@ -10,7 +10,7 @@
 #
 # It's strongly recommended that you check this file into your version control system.
 
-ActiveRecord::Schema.define(version: 2019_06_27_222826) do
+ActiveRecord::Schema.define(version: 2019_07_06_233204) do
 
   # These are extensions that must be enabled in order to support this database
   enable_extension "plpgsql"
@@ -647,17 +647,6 @@ ActiveRecord::Schema.define(version: 2019_06_27_222826) do
     t.index ["tag_id", "status_id"], name: "index_statuses_tags_on_tag_id_and_status_id", unique: true
   end
 
-  create_table "stream_entries", force: :cascade do |t|
-    t.bigint "activity_id"
-    t.string "activity_type"
-    t.datetime "created_at", null: false
-    t.datetime "updated_at", null: false
-    t.boolean "hidden", default: false, null: false
-    t.bigint "account_id"
-    t.index ["account_id", "activity_type", "id"], name: "index_stream_entries_on_account_id_and_activity_type_and_id"
-    t.index ["activity_id", "activity_type"], name: "index_stream_entries_on_activity_id_and_activity_type"
-  end
-
   create_table "subscriptions", force: :cascade do |t|
     t.string "callback_url", default: "", null: false
     t.string "secret"
@@ -831,7 +820,6 @@ ActiveRecord::Schema.define(version: 2019_06_27_222826) do
   add_foreign_key "statuses", "statuses", column: "reblog_of_id", on_delete: :cascade
   add_foreign_key "statuses_tags", "statuses", on_delete: :cascade
   add_foreign_key "statuses_tags", "tags", name: "fk_3081861e21", on_delete: :cascade
-  add_foreign_key "stream_entries", "accounts", name: "fk_5659b17554", on_delete: :cascade
   add_foreign_key "subscriptions", "accounts", name: "fk_9847d1cbb5", on_delete: :cascade
   add_foreign_key "tombstones", "accounts", on_delete: :cascade
   add_foreign_key "user_invite_requests", "users", on_delete: :cascade
diff --git a/spec/controllers/accounts_controller_spec.rb b/spec/controllers/accounts_controller_spec.rb
index b728d719f..3d2a0665d 100644
--- a/spec/controllers/accounts_controller_spec.rb
+++ b/spec/controllers/accounts_controller_spec.rb
@@ -48,37 +48,6 @@ RSpec.describe AccountsController, type: :controller do
       end
     end
 
-    context 'atom' do
-      let(:format) { 'atom' }
-      let(:content_type) { 'application/atom+xml' }
-
-      shared_examples 'responsed streams' do
-        it 'assigns @entries' do
-          entries = assigns(:entries).to_a
-          expect(entries.size).to eq expected_statuses.size
-          entries.each.zip(expected_statuses.each) do |entry, expected_status|
-            expect(entry.status).to eq expected_status
-          end
-        end
-      end
-
-      include_examples 'responses'
-
-      context 'without max_id nor since_id' do
-        let(:expected_statuses) { [status7, status6, status5, status4, status3, status2, status1] }
-
-        include_examples 'responsed streams'
-      end
-
-      context 'with max_id and since_id' do
-        let(:max_id) { status4.stream_entry.id }
-        let(:since_id) { status1.stream_entry.id }
-        let(:expected_statuses) { [status3, status2] }
-
-        include_examples 'responsed streams'
-      end
-    end
-
     context 'activitystreams2' do
       let(:format) { 'json' }
       let(:content_type) { 'application/activity+json' }
diff --git a/spec/controllers/api/oembed_controller_spec.rb b/spec/controllers/api/oembed_controller_spec.rb
index 7fee15a35..b9082bde1 100644
--- a/spec/controllers/api/oembed_controller_spec.rb
+++ b/spec/controllers/api/oembed_controller_spec.rb
@@ -9,7 +9,7 @@ RSpec.describe Api::OEmbedController, type: :controller do
   describe 'GET #show' do
     before do
       request.host = Rails.configuration.x.local_domain
-      get :show, params: { url: account_stream_entry_url(alice, status.stream_entry) }, format: :json
+      get :show, params: { url: short_account_status_url(alice, status) }, format: :json
     end
 
     it 'returns http success' do
diff --git a/spec/controllers/application_controller_spec.rb b/spec/controllers/application_controller_spec.rb
index 33cc71087..27946b60f 100644
--- a/spec/controllers/application_controller_spec.rb
+++ b/spec/controllers/application_controller_spec.rb
@@ -360,9 +360,5 @@ describe ApplicationController, type: :controller do
     context 'Status' do
       include_examples 'cacheable', :status, Status
     end
-
-    context 'StreamEntry' do
-      include_examples 'receives :with_includes', :stream_entry, StreamEntry
-    end
   end
 end
diff --git a/spec/controllers/statuses_controller_spec.rb b/spec/controllers/statuses_controller_spec.rb
index 1bb6636c6..95e5c363c 100644
--- a/spec/controllers/statuses_controller_spec.rb
+++ b/spec/controllers/statuses_controller_spec.rb
@@ -55,18 +55,6 @@ describe StatusesController do
         expect(assigns(:status)).to eq status
       end
 
-      it 'assigns @stream_entry' do
-        status = Fabricate(:status)
-        get :show, params: { account_username: status.account.username, id: status.id }
-        expect(assigns(:stream_entry)).to eq status.stream_entry
-      end
-
-      it 'assigns @type' do
-        status = Fabricate(:status)
-        get :show, params: { account_username: status.account.username, id: status.id }
-        expect(assigns(:type)).to eq 'status'
-      end
-
       it 'assigns @ancestors for ancestors of the status if it is a reply' do
         ancestor = Fabricate(:status)
         status = Fabricate(:status, in_reply_to_id: ancestor.id)
@@ -135,10 +123,10 @@ describe StatusesController do
         expect(response).to have_http_status(200)
       end
 
-      it 'renders stream_entries/show' do
+      it 'renders statuses/show' do
         status = Fabricate(:status)
         get :show, params: { account_username: status.account.username, id: status.id }
-        expect(response).to render_template 'stream_entries/show'
+        expect(response).to render_template 'statuses/show'
       end
     end
   end
diff --git a/spec/controllers/stream_entries_controller_spec.rb b/spec/controllers/stream_entries_controller_spec.rb
deleted file mode 100644
index eb7fdf9d7..000000000
--- a/spec/controllers/stream_entries_controller_spec.rb
+++ /dev/null
@@ -1,95 +0,0 @@
-require 'rails_helper'
-
-RSpec.describe StreamEntriesController, type: :controller do
-  render_views
-
-  shared_examples 'before_action' do |route|
-    context 'when account is not suspended and stream_entry is available' do
-      it 'assigns instance variables' do
-        status = Fabricate(:status)
-
-        get route, params: { account_username: status.account.username, id: status.stream_entry.id }
-
-        expect(assigns(:account)).to eq status.account
-        expect(assigns(:stream_entry)).to eq status.stream_entry
-        expect(assigns(:type)).to eq 'status'
-      end
-
-      it 'sets Link headers' do
-        alice = Fabricate(:account, username: 'alice')
-        status = Fabricate(:status, account: alice)
-
-        get route, params: { account_username: alice.username, id: status.stream_entry.id }
-
-        expect(response.headers['Link'].to_s).to eq "<http://test.host/users/alice/updates/#{status.stream_entry.id}.atom>; rel=\"alternate\"; type=\"application/atom+xml\", <https://cb6e6126.ngrok.io/users/alice/statuses/#{status.id}>; rel=\"alternate\"; type=\"application/activity+json\""
-      end
-    end
-
-    context 'when account is suspended' do
-      it 'returns http status 410' do
-        account = Fabricate(:account, suspended: true)
-        status = Fabricate(:status, account: account)
-
-        get route, params: { account_username: account.username, id: status.stream_entry.id }
-
-        expect(response).to have_http_status(410)
-      end
-    end
-
-    context 'when activity is nil' do
-      it 'raises ActiveRecord::RecordNotFound' do
-        account = Fabricate(:account)
-        stream_entry = Fabricate.build(:stream_entry, account: account, activity: nil, activity_type: 'Status')
-        stream_entry.save!(validate: false)
-
-        get route, params: { account_username: account.username, id: stream_entry.id }
-
-        expect(response).to have_http_status(404)
-      end
-    end
-
-    context 'when it is hidden and it is not permitted' do
-      it 'raises ActiveRecord::RecordNotFound' do
-        status = Fabricate(:status)
-        user = Fabricate(:user)
-        status.account.block!(user.account)
-        status.stream_entry.update!(hidden: true)
-
-        sign_in(user)
-        get route, params: { account_username: status.account.username, id: status.stream_entry.id }
-
-        expect(response).to have_http_status(404)
-      end
-    end
-  end
-
-  describe 'GET #show' do
-    include_examples 'before_action', :show
-
-    it 'redirects to status page' do
-      status = Fabricate(:status)
-
-      get :show, params: { account_username: status.account.username, id: status.stream_entry.id }
-
-      expect(response).to redirect_to(short_account_status_url(status.account, status))
-    end
-
-    it 'returns http success with Atom' do
-      status = Fabricate(:status)
-      get :show, params: { account_username: status.account.username, id: status.stream_entry.id }, format: 'atom'
-      expect(response).to have_http_status(200)
-    end
-  end
-
-  describe 'GET #embed' do
-    include_examples 'before_action', :embed
-
-    it 'redirects to new embed page' do
-      status = Fabricate(:status)
-
-      get :embed, params: { account_username: status.account.username, id: status.stream_entry.id }
-
-      expect(response).to redirect_to(embed_short_account_status_url(status.account, status))
-    end
-  end
-end
diff --git a/spec/fabricators/stream_entry_fabricator.rb b/spec/fabricators/stream_entry_fabricator.rb
deleted file mode 100644
index f33822c7c..000000000
--- a/spec/fabricators/stream_entry_fabricator.rb
+++ /dev/null
@@ -1,5 +0,0 @@
-Fabricator(:stream_entry) do
-  account
-  activity { Fabricate(:status) }
-  hidden { [true, false].sample }
-end
diff --git a/spec/helpers/admin/account_moderation_notes_helper_spec.rb b/spec/helpers/admin/account_moderation_notes_helper_spec.rb
index c07f6c4b8..ddfe8b46f 100644
--- a/spec/helpers/admin/account_moderation_notes_helper_spec.rb
+++ b/spec/helpers/admin/account_moderation_notes_helper_spec.rb
@@ -3,7 +3,7 @@
 require 'rails_helper'
 
 RSpec.describe Admin::AccountModerationNotesHelper, type: :helper do
-  include StreamEntriesHelper
+  include StatusesHelper
 
   describe '#admin_account_link_to' do
     context 'account is nil' do
diff --git a/spec/helpers/statuses_helper_spec.rb b/spec/helpers/statuses_helper_spec.rb
new file mode 100644
index 000000000..510955a2f
--- /dev/null
+++ b/spec/helpers/statuses_helper_spec.rb
@@ -0,0 +1,224 @@
+require 'rails_helper'
+
+RSpec.describe StatusesHelper, type: :helper do
+  describe '#display_name' do
+    it 'uses the display name when it exists' do
+      account = Account.new(display_name: "Display", username: "Username")
+
+      expect(helper.display_name(account)).to eq "Display"
+    end
+
+    it 'uses the username when display name is nil' do
+      account = Account.new(display_name: nil, username: "Username")
+
+      expect(helper.display_name(account)).to eq "Username"
+    end
+  end
+
+  describe '#stream_link_target' do
+    it 'returns nil if it is not an embedded view' do
+      set_not_embedded_view
+
+      expect(helper.stream_link_target).to be_nil
+    end
+
+    it 'returns _blank if it is an embedded view' do
+      set_embedded_view
+
+      expect(helper.stream_link_target).to eq '_blank'
+    end
+  end
+
+  describe '#acct' do
+    it 'is fully qualified for embedded local accounts' do
+      allow(Rails.configuration.x).to receive(:local_domain).and_return('local_domain')
+      set_embedded_view
+      account = Account.new(domain: nil, username: 'user')
+
+      acct = helper.acct(account)
+
+      expect(acct).to eq '@user@local_domain'
+    end
+
+    it 'is fully qualified for embedded foreign accounts' do
+      set_embedded_view
+      account = Account.new(domain: 'foreign_server.com', username: 'user')
+
+      acct = helper.acct(account)
+
+      expect(acct).to eq '@user@foreign_server.com'
+    end
+
+    it 'is fully qualified for non embedded foreign accounts' do
+      set_not_embedded_view
+      account = Account.new(domain: 'foreign_server.com', username: 'user')
+
+      acct = helper.acct(account)
+
+      expect(acct).to eq '@user@foreign_server.com'
+    end
+
+    it 'is fully qualified for non embedded local accounts' do
+      allow(Rails.configuration.x).to receive(:local_domain).and_return('local_domain')
+      set_not_embedded_view
+      account = Account.new(domain: nil, username: 'user')
+
+      acct = helper.acct(account)
+
+      expect(acct).to eq '@user@local_domain'
+    end
+  end
+
+  def set_not_embedded_view
+    params[:controller] = "not_#{StatusesHelper::EMBEDDED_CONTROLLER}"
+    params[:action] = "not_#{StatusesHelper::EMBEDDED_ACTION}"
+  end
+
+  def set_embedded_view
+    params[:controller] = StatusesHelper::EMBEDDED_CONTROLLER
+    params[:action] = StatusesHelper::EMBEDDED_ACTION
+  end
+
+  describe '#style_classes' do
+    it do
+      status = double(reblog?: false)
+      classes = helper.style_classes(status, false, false, false)
+
+      expect(classes).to eq 'entry'
+    end
+
+    it do
+      status = double(reblog?: true)
+      classes = helper.style_classes(status, false, false, false)
+
+      expect(classes).to eq 'entry entry-reblog'
+    end
+
+    it do
+      status = double(reblog?: false)
+      classes = helper.style_classes(status, true, false, false)
+
+      expect(classes).to eq 'entry entry-predecessor'
+    end
+
+    it do
+      status = double(reblog?: false)
+      classes = helper.style_classes(status, false, true, false)
+
+      expect(classes).to eq 'entry entry-successor'
+    end
+
+    it do
+      status = double(reblog?: false)
+      classes = helper.style_classes(status, false, false, true)
+
+      expect(classes).to eq 'entry entry-center'
+    end
+
+    it do
+      status = double(reblog?: true)
+      classes = helper.style_classes(status, true, true, true)
+
+      expect(classes).to eq 'entry entry-predecessor entry-reblog entry-successor entry-center'
+    end
+  end
+
+  describe '#microformats_classes' do
+    it do
+      status = double(reblog?: false)
+      classes = helper.microformats_classes(status, false, false)
+
+      expect(classes).to eq ''
+    end
+
+    it do
+      status = double(reblog?: false)
+      classes = helper.microformats_classes(status, true, false)
+
+      expect(classes).to eq 'p-in-reply-to'
+    end
+
+    it do
+      status = double(reblog?: false)
+      classes = helper.microformats_classes(status, false, true)
+
+      expect(classes).to eq 'p-comment'
+    end
+
+    it do
+      status = double(reblog?: true)
+      classes = helper.microformats_classes(status, true, false)
+
+      expect(classes).to eq 'p-in-reply-to p-repost-of'
+    end
+
+    it do
+      status = double(reblog?: true)
+      classes = helper.microformats_classes(status, true, true)
+
+      expect(classes).to eq 'p-in-reply-to p-repost-of p-comment'
+    end
+  end
+
+  describe '#microformats_h_class' do
+    it do
+      status = double(reblog?: false)
+      css_class = helper.microformats_h_class(status, false, false, false)
+
+      expect(css_class).to eq 'h-entry'
+    end
+
+    it do
+      status = double(reblog?: true)
+      css_class = helper.microformats_h_class(status, false, false, false)
+
+      expect(css_class).to eq 'h-cite'
+    end
+
+    it do
+      status = double(reblog?: false)
+      css_class = helper.microformats_h_class(status, true, false, false)
+
+      expect(css_class).to eq 'h-cite'
+    end
+
+    it do
+      status = double(reblog?: false)
+      css_class = helper.microformats_h_class(status, false, true, false)
+
+      expect(css_class).to eq 'h-cite'
+    end
+
+    it do
+      status = double(reblog?: false)
+      css_class = helper.microformats_h_class(status, false, false, true)
+
+      expect(css_class).to eq ''
+    end
+
+    it do
+      status = double(reblog?: true)
+      css_class = helper.microformats_h_class(status, true, true, true)
+
+      expect(css_class).to eq 'h-cite'
+    end
+  end
+
+  describe '#rtl?' do
+    it 'is false if text is empty' do
+      expect(helper).not_to be_rtl ''
+    end
+
+    it 'is false if there are no right to left characters' do
+      expect(helper).not_to be_rtl 'hello world'
+    end
+
+    it 'is false if right to left characters are fewer than 1/3 of total text' do
+      expect(helper).not_to be_rtl 'hello ݟ world'
+    end
+
+    it 'is true if right to left characters are greater than 1/3 of total text' do
+      expect(helper).to be_rtl 'aaݟaaݟ'
+    end
+  end
+end
diff --git a/spec/helpers/stream_entries_helper_spec.rb b/spec/helpers/stream_entries_helper_spec.rb
deleted file mode 100644
index 845b9974e..000000000
--- a/spec/helpers/stream_entries_helper_spec.rb
+++ /dev/null
@@ -1,224 +0,0 @@
-require 'rails_helper'
-
-RSpec.describe StreamEntriesHelper, type: :helper do
-  describe '#display_name' do
-    it 'uses the display name when it exists' do
-      account = Account.new(display_name: "Display", username: "Username")
-
-      expect(helper.display_name(account)).to eq "Display"
-    end
-
-    it 'uses the username when display name is nil' do
-      account = Account.new(display_name: nil, username: "Username")
-
-      expect(helper.display_name(account)).to eq "Username"
-    end
-  end
-
-  describe '#stream_link_target' do
-    it 'returns nil if it is not an embedded view' do
-      set_not_embedded_view
-
-      expect(helper.stream_link_target).to be_nil
-    end
-
-    it 'returns _blank if it is an embedded view' do
-      set_embedded_view
-
-      expect(helper.stream_link_target).to eq '_blank'
-    end
-  end
-
-  describe '#acct' do
-    it 'is fully qualified for embedded local accounts' do
-      allow(Rails.configuration.x).to receive(:local_domain).and_return('local_domain')
-      set_embedded_view
-      account = Account.new(domain: nil, username: 'user')
-
-      acct = helper.acct(account)
-
-      expect(acct).to eq '@user@local_domain'
-    end
-
-    it 'is fully qualified for embedded foreign accounts' do
-      set_embedded_view
-      account = Account.new(domain: 'foreign_server.com', username: 'user')
-
-      acct = helper.acct(account)
-
-      expect(acct).to eq '@user@foreign_server.com'
-    end
-
-    it 'is fully qualified for non embedded foreign accounts' do
-      set_not_embedded_view
-      account = Account.new(domain: 'foreign_server.com', username: 'user')
-
-      acct = helper.acct(account)
-
-      expect(acct).to eq '@user@foreign_server.com'
-    end
-
-    it 'is fully qualified for non embedded local accounts' do
-      allow(Rails.configuration.x).to receive(:local_domain).and_return('local_domain')
-      set_not_embedded_view
-      account = Account.new(domain: nil, username: 'user')
-
-      acct = helper.acct(account)
-
-      expect(acct).to eq '@user@local_domain'
-    end
-  end
-
-  def set_not_embedded_view
-    params[:controller] = "not_#{StreamEntriesHelper::EMBEDDED_CONTROLLER}"
-    params[:action] = "not_#{StreamEntriesHelper::EMBEDDED_ACTION}"
-  end
-
-  def set_embedded_view
-    params[:controller] = StreamEntriesHelper::EMBEDDED_CONTROLLER
-    params[:action] = StreamEntriesHelper::EMBEDDED_ACTION
-  end
-
-  describe '#style_classes' do
-    it do
-      status = double(reblog?: false)
-      classes = helper.style_classes(status, false, false, false)
-
-      expect(classes).to eq 'entry'
-    end
-
-    it do
-      status = double(reblog?: true)
-      classes = helper.style_classes(status, false, false, false)
-
-      expect(classes).to eq 'entry entry-reblog'
-    end
-
-    it do
-      status = double(reblog?: false)
-      classes = helper.style_classes(status, true, false, false)
-
-      expect(classes).to eq 'entry entry-predecessor'
-    end
-
-    it do
-      status = double(reblog?: false)
-      classes = helper.style_classes(status, false, true, false)
-
-      expect(classes).to eq 'entry entry-successor'
-    end
-
-    it do
-      status = double(reblog?: false)
-      classes = helper.style_classes(status, false, false, true)
-
-      expect(classes).to eq 'entry entry-center'
-    end
-
-    it do
-      status = double(reblog?: true)
-      classes = helper.style_classes(status, true, true, true)
-
-      expect(classes).to eq 'entry entry-predecessor entry-reblog entry-successor entry-center'
-    end
-  end
-
-  describe '#microformats_classes' do
-    it do
-      status = double(reblog?: false)
-      classes = helper.microformats_classes(status, false, false)
-
-      expect(classes).to eq ''
-    end
-
-    it do
-      status = double(reblog?: false)
-      classes = helper.microformats_classes(status, true, false)
-
-      expect(classes).to eq 'p-in-reply-to'
-    end
-
-    it do
-      status = double(reblog?: false)
-      classes = helper.microformats_classes(status, false, true)
-
-      expect(classes).to eq 'p-comment'
-    end
-
-    it do
-      status = double(reblog?: true)
-      classes = helper.microformats_classes(status, true, false)
-
-      expect(classes).to eq 'p-in-reply-to p-repost-of'
-    end
-
-    it do
-      status = double(reblog?: true)
-      classes = helper.microformats_classes(status, true, true)
-
-      expect(classes).to eq 'p-in-reply-to p-repost-of p-comment'
-    end
-  end
-
-  describe '#microformats_h_class' do
-    it do
-      status = double(reblog?: false)
-      css_class = helper.microformats_h_class(status, false, false, false)
-
-      expect(css_class).to eq 'h-entry'
-    end
-
-    it do
-      status = double(reblog?: true)
-      css_class = helper.microformats_h_class(status, false, false, false)
-
-      expect(css_class).to eq 'h-cite'
-    end
-
-    it do
-      status = double(reblog?: false)
-      css_class = helper.microformats_h_class(status, true, false, false)
-
-      expect(css_class).to eq 'h-cite'
-    end
-
-    it do
-      status = double(reblog?: false)
-      css_class = helper.microformats_h_class(status, false, true, false)
-
-      expect(css_class).to eq 'h-cite'
-    end
-
-    it do
-      status = double(reblog?: false)
-      css_class = helper.microformats_h_class(status, false, false, true)
-
-      expect(css_class).to eq ''
-    end
-
-    it do
-      status = double(reblog?: true)
-      css_class = helper.microformats_h_class(status, true, true, true)
-
-      expect(css_class).to eq 'h-cite'
-    end
-  end
-
-  describe '#rtl?' do
-    it 'is false if text is empty' do
-      expect(helper).not_to be_rtl ''
-    end
-
-    it 'is false if there are no right to left characters' do
-      expect(helper).not_to be_rtl 'hello world'
-    end
-
-    it 'is false if right to left characters are fewer than 1/3 of total text' do
-      expect(helper).not_to be_rtl 'hello ݟ world'
-    end
-
-    it 'is true if right to left characters are greater than 1/3 of total text' do
-      expect(helper).to be_rtl 'aaݟaaݟ'
-    end
-  end
-end
diff --git a/spec/lib/activitypub/tag_manager_spec.rb b/spec/lib/activitypub/tag_manager_spec.rb
index 6d246629e..1c5c6f0ed 100644
--- a/spec/lib/activitypub/tag_manager_spec.rb
+++ b/spec/lib/activitypub/tag_manager_spec.rb
@@ -143,12 +143,6 @@ RSpec.describe ActivityPub::TagManager do
       expect(subject.uri_to_resource(OStatus::TagManager.instance.uri_for(status), Status)).to eq status
     end
 
-    it 'returns the local status for OStatus StreamEntry URL' do
-      status = Fabricate(:status)
-      stream_entry_url = account_stream_entry_url(status.account, status.stream_entry)
-      expect(subject.uri_to_resource(stream_entry_url, Status)).to eq status
-    end
-
     it 'returns the remote status by matching URI without fragment part' do
       status = Fabricate(:status, uri: 'https://example.com/123')
       expect(subject.uri_to_resource('https://example.com/123#456', Status)).to eq status
diff --git a/spec/lib/ostatus/atom_serializer_spec.rb b/spec/lib/ostatus/atom_serializer_spec.rb
deleted file mode 100644
index 74ab7576f..000000000
--- a/spec/lib/ostatus/atom_serializer_spec.rb
+++ /dev/null
@@ -1,1415 +0,0 @@
-require 'rails_helper'
-
-RSpec.describe OStatus::AtomSerializer do
-  shared_examples 'follow request salmon' do
-    it 'appends author element with account' do
-      account = Fabricate(:account, domain: nil, username: 'username')
-      follow_request = Fabricate(:follow_request, account: account)
-
-      follow_request_salmon = serialize(follow_request)
-
-      expect(follow_request_salmon.author.id.text).to eq 'https://cb6e6126.ngrok.io/users/username'
-    end
-
-    it 'appends activity:object-type element with activity type' do
-      follow_request = Fabricate(:follow_request)
-
-      follow_request_salmon = serialize(follow_request)
-
-      object_type = follow_request_salmon.nodes.find { |node| node.name == 'activity:object-type' }
-      expect(object_type.text).to eq OStatus::TagManager::TYPES[:activity]
-    end
-
-    it 'appends activity:verb element with request_friend type' do
-      follow_request = Fabricate(:follow_request)
-
-      follow_request_salmon = serialize(follow_request)
-
-      verb = follow_request_salmon.nodes.find { |node| node.name == 'activity:verb' }
-      expect(verb.text).to eq OStatus::TagManager::VERBS[:request_friend]
-    end
-
-    it 'appends activity:object with target account' do
-      target_account = Fabricate(:account, domain: 'domain.test', uri: 'https://domain.test/id')
-      follow_request = Fabricate(:follow_request, target_account: target_account)
-
-      follow_request_salmon = serialize(follow_request)
-
-      object = follow_request_salmon.nodes.find { |node| node.name == 'activity:object' }
-      expect(object.id.text).to eq 'https://domain.test/id'
-    end
-  end
-
-  shared_examples 'namespaces' do
-    it 'adds namespaces' do
-      element = serialize
-
-      expect(element['xmlns']).to eq OStatus::TagManager::XMLNS
-      expect(element['xmlns:thr']).to eq OStatus::TagManager::THR_XMLNS
-      expect(element['xmlns:activity']).to eq OStatus::TagManager::AS_XMLNS
-      expect(element['xmlns:poco']).to eq OStatus::TagManager::POCO_XMLNS
-      expect(element['xmlns:media']).to eq OStatus::TagManager::MEDIA_XMLNS
-      expect(element['xmlns:ostatus']).to eq OStatus::TagManager::OS_XMLNS
-      expect(element['xmlns:mastodon']).to eq OStatus::TagManager::MTDN_XMLNS
-    end
-  end
-
-  shared_examples 'no namespaces' do
-    it 'does not add namespaces' do
-      expect(serialize['xmlns']).to eq nil
-    end
-  end
-
-  shared_examples 'status attributes' do
-    it 'appends summary element with spoiler text if present' do
-      status = Fabricate(:status, language: :ca, spoiler_text: 'spoiler text')
-
-      element = serialize(status)
-
-      summary = element.summary
-      expect(summary['xml:lang']).to eq 'ca'
-      expect(summary.text).to eq 'spoiler text'
-    end
-
-    it 'does not append summary element with spoiler text if not present' do
-      status = Fabricate(:status, spoiler_text: '')
-      element = serialize(status)
-      element.nodes.each { |node| expect(node.name).not_to eq 'summary' }
-    end
-
-    it 'appends content element with formatted status' do
-      status = Fabricate(:status, language: :ca, text: 'text')
-
-      element = serialize(status)
-
-      content = element.content
-      expect(content[:type]).to eq 'html'
-      expect(content['xml:lang']).to eq 'ca'
-      expect(content.text).to eq '<p>text</p>'
-    end
-
-    it 'appends link elements for mentioned accounts' do
-      account = Fabricate(:account, username: 'username')
-      status = Fabricate(:status)
-      Fabricate(:mention, account: account, status: status)
-
-      element = serialize(status)
-
-      mentioned = element.nodes.find do |node|
-        node.name == 'link' &&
-          node[:rel] == 'mentioned' &&
-          node['ostatus:object-type'] == OStatus::TagManager::TYPES[:person]
-      end
-
-      expect(mentioned[:href]).to eq 'https://cb6e6126.ngrok.io/users/username'
-    end
-
-    it 'appends link elements for emojis' do
-      Fabricate(:custom_emoji)
-
-      status  = Fabricate(:status, text: ':coolcat:')
-      element = serialize(status)
-      emoji   = element.nodes.find { |node| node.name == 'link' && node[:rel] == 'emoji' }
-
-      expect(emoji[:name]).to eq 'coolcat'
-      expect(emoji[:href]).to_not be_blank
-    end
-  end
-
-  describe 'render' do
-    it 'returns XML with emojis' do
-      element = Ox::Element.new('tag')
-      element << '💩'
-      xml = OStatus::AtomSerializer.render(element)
-
-      expect(xml).to eq "<?xml version=\"1.0\"?>\n<tag>💩</tag>\n"
-    end
-
-    it 'returns XML, stripping invalid characters like \b and \v' do
-      element = Ox::Element.new('tag')
-      element << "im l33t\b haxo\b\vr"
-      xml = OStatus::AtomSerializer.render(element)
-
-      expect(xml).to eq "<?xml version=\"1.0\"?>\n<tag>im l33t haxor</tag>\n"
-    end
-  end
-
-  describe '#author' do
-    context 'when note is present' do
-      it 'appends poco:note element with note for local account' do
-        account = Fabricate(:account, domain: nil, note: '<p>note</p>')
-
-        author = OStatus::AtomSerializer.new.author(account)
-
-        note = author.nodes.find { |node| node.name == 'poco:note' }
-        expect(note.text).to eq '<p>note</p>'
-      end
-
-      it 'appends poco:note element with tags-stripped note for remote account' do
-        account = Fabricate(:account, domain: 'remote', note: '<p>note</p>')
-
-        author = OStatus::AtomSerializer.new.author(account)
-
-        note = author.nodes.find { |node| node.name == 'poco:note' }
-        expect(note.text).to eq 'note'
-      end
-
-      it 'appends summary element with type attribute and simplified note if present' do
-        account = Fabricate(:account, note: 'note')
-        author = OStatus::AtomSerializer.new.author(account)
-        expect(author.summary.text).to eq '<p>note</p>'
-        expect(author.summary[:type]).to eq 'html'
-      end
-    end
-
-    context 'when note is not present' do
-      it 'does not append poco:note element' do
-        account = Fabricate(:account, note: '')
-        author = OStatus::AtomSerializer.new.author(account)
-        author.nodes.each { |node| expect(node.name).not_to eq 'poco:note' }
-      end
-
-      it 'does not append summary element' do
-        account = Fabricate(:account, note: '')
-        author = OStatus::AtomSerializer.new.author(account)
-        author.nodes.each { |node| expect(node.name).not_to eq 'summary' }
-      end
-    end
-
-    it 'returns author element' do
-      account = Fabricate(:account)
-      author = OStatus::AtomSerializer.new.author(account)
-      expect(author.name).to eq 'author'
-    end
-
-    it 'appends activity:object-type element with person type' do
-      account = Fabricate(:account, domain: nil, username: 'username')
-
-      author = OStatus::AtomSerializer.new.author(account)
-
-      object_type = author.nodes.find { |node| node.name == 'activity:object-type' }
-      expect(object_type.text).to eq OStatus::TagManager::TYPES[:person]
-    end
-
-    it 'appends email element with username and domain for local account' do
-      account = Fabricate(:account, username: 'username')
-      author = OStatus::AtomSerializer.new.author(account)
-      expect(author.email.text).to eq 'username@cb6e6126.ngrok.io'
-    end
-
-    it 'appends email element with username and domain for remote user' do
-      account = Fabricate(:account, domain: 'domain', username: 'username')
-      author = OStatus::AtomSerializer.new.author(account)
-      expect(author.email.text).to eq 'username@domain'
-    end
-
-    it 'appends link element for an alternative' do
-      account = Fabricate(:account, domain: nil, username: 'username')
-
-      author = OStatus::AtomSerializer.new.author(account)
-
-      link = author.nodes.find { |node| node.name == 'link' && node[:rel] == 'alternate' && node[:type] == 'text/html' }
-      expect(link[:type]).to eq 'text/html'
-      expect(link[:rel]).to eq 'alternate'
-      expect(link[:href]).to eq 'https://cb6e6126.ngrok.io/@username'
-    end
-
-    it 'has link element for avatar if present' do
-      account = Fabricate(:account, avatar: attachment_fixture('avatar.gif'))
-
-      author = OStatus::AtomSerializer.new.author(account)
-
-      link = author.nodes.find { |node| node.name == 'link' && node[:rel] == 'avatar' }
-      expect(link[:type]).to eq 'image/gif'
-      expect(link['media:width']).to eq '120'
-      expect(link['media:height']).to eq '120'
-      expect(link[:href]).to match  /^https:\/\/cb6e6126.ngrok.io\/system\/accounts\/avatars\/.+\/original\/avatar.gif/
-    end
-
-    it 'does not have link element for avatar if not present' do
-      account = Fabricate(:account, avatar: nil)
-
-      author = OStatus::AtomSerializer.new.author(account)
-
-      author.nodes.each do |node|
-        expect(node[:rel]).not_to eq 'avatar' if node.name == 'link'
-      end
-    end
-
-    it 'appends link element for header if present' do
-      account = Fabricate(:account, header: attachment_fixture('avatar.gif'))
-
-      author = OStatus::AtomSerializer.new.author(account)
-
-      link = author.nodes.find { |node| node.name == 'link' && node[:rel] == 'header' }
-      expect(link[:type]).to eq 'image/gif'
-      expect(link['media:width']).to eq '700'
-      expect(link['media:height']).to eq '335'
-      expect(link[:href]).to match  /^https:\/\/cb6e6126.ngrok.io\/system\/accounts\/headers\/.+\/original\/avatar.gif/
-    end
-
-    it 'does not append link element for header if not present' do
-      account = Fabricate(:account, header: nil)
-
-      author = OStatus::AtomSerializer.new.author(account)
-
-      author.nodes.each do |node|
-        expect(node[:rel]).not_to eq 'header' if node.name == 'link'
-      end
-    end
-
-    it 'appends poco:displayName element with display name if present' do
-      account = Fabricate(:account, display_name: 'display name')
-
-      author = OStatus::AtomSerializer.new.author(account)
-
-      display_name = author.nodes.find { |node| node.name == 'poco:displayName' }
-      expect(display_name.text).to eq 'display name'
-    end
-
-    it 'does not append poco:displayName element with display name if not present' do
-      account = Fabricate(:account, display_name: '')
-      author = OStatus::AtomSerializer.new.author(account)
-      author.nodes.each { |node| expect(node.name).not_to eq 'poco:displayName' }
-    end
-
-    it "appends mastodon:scope element with 'private' if locked" do
-      account = Fabricate(:account, locked: true)
-
-      author = OStatus::AtomSerializer.new.author(account)
-
-      scope = author.nodes.find { |node| node.name == 'mastodon:scope' }
-      expect(scope.text).to eq 'private'
-    end
-
-    it "appends mastodon:scope element with 'public' if unlocked" do
-      account = Fabricate(:account, locked: false)
-
-      author = OStatus::AtomSerializer.new.author(account)
-
-      scope = author.nodes.find { |node| node.name == 'mastodon:scope' }
-      expect(scope.text).to eq 'public'
-    end
-
-    it 'includes URI' do
-      account = Fabricate(:account, domain: nil, username: 'username')
-
-      author = OStatus::AtomSerializer.new.author(account)
-
-      expect(author.id.text).to eq 'https://cb6e6126.ngrok.io/users/username'
-      expect(author.uri.text).to eq 'https://cb6e6126.ngrok.io/users/username'
-    end
-
-    it 'includes username' do
-      account = Fabricate(:account, username: 'username')
-
-      author = OStatus::AtomSerializer.new.author(account)
-
-      name = author.nodes.find { |node| node.name == 'name' }
-      username = author.nodes.find { |node| node.name == 'poco:preferredUsername' }
-      expect(name.text).to eq 'username'
-      expect(username.text).to eq 'username'
-    end
-  end
-
-  describe '#entry' do
-    shared_examples 'not root' do
-      include_examples 'no namespaces' do
-        def serialize
-          subject
-        end
-      end
-
-      it 'does not append author element' do
-        subject.nodes.each { |node| expect(node.name).not_to eq 'author' }
-      end
-    end
-
-    context 'it is root' do
-      include_examples 'namespaces' do
-        def serialize
-          stream_entry = Fabricate(:stream_entry)
-          OStatus::AtomSerializer.new.entry(stream_entry, true)
-        end
-      end
-
-      it 'appends author element' do
-        account = Fabricate(:account, username: 'username')
-        status = Fabricate(:status, account: account)
-
-        entry = OStatus::AtomSerializer.new.entry(status.stream_entry, true)
-
-        expect(entry.author.id.text).to eq 'https://cb6e6126.ngrok.io/users/username'
-      end
-    end
-
-    context 'if status is present' do
-      include_examples 'status attributes' do
-        def serialize(status)
-          OStatus::AtomSerializer.new.entry(status.stream_entry, true)
-        end
-      end
-
-      it 'appends link element for the public collection if status is publicly visible' do
-        status = Fabricate(:status, visibility: :public)
-
-        entry = OStatus::AtomSerializer.new.entry(status.stream_entry)
-
-        mentioned_person = entry.nodes.find do |node|
-          node.name == 'link' &&
-          node[:rel] == 'mentioned' &&
-          node['ostatus:object-type'] == OStatus::TagManager::TYPES[:collection]
-        end
-        expect(mentioned_person[:href]).to eq OStatus::TagManager::COLLECTIONS[:public]
-      end
-
-      it 'does not append link element for the public collection if status is not publicly visible' do
-        status = Fabricate(:status, visibility: :private)
-
-        entry = OStatus::AtomSerializer.new.entry(status.stream_entry)
-
-        entry.nodes.each do |node|
-          if node.name == 'link' &&
-             node[:rel] == 'mentioned' &&
-             node['ostatus:object-type'] == OStatus::TagManager::TYPES[:collection]
-            expect(mentioned_collection[:href]).not_to eq OStatus::TagManager::COLLECTIONS[:public]
-          end
-        end
-      end
-
-      it 'appends category elements for tags' do
-        tag = Fabricate(:tag, name: 'tag')
-        status = Fabricate(:status, tags: [ tag ])
-
-        entry = OStatus::AtomSerializer.new.entry(status.stream_entry)
-
-        expect(entry.category[:term]).to eq 'tag'
-      end
-
-      it 'appends link elements for media attachments' do
-        file = attachment_fixture('attachment.jpg')
-        media_attachment = Fabricate(:media_attachment, file: file)
-        status = Fabricate(:status, media_attachments: [ media_attachment ])
-
-        entry = OStatus::AtomSerializer.new.entry(status.stream_entry)
-
-        enclosure = entry.nodes.find { |node| node.name == 'link' && node[:rel] == 'enclosure' }
-        expect(enclosure[:type]).to eq 'image/jpeg'
-        expect(enclosure[:href]).to match /^https:\/\/cb6e6126.ngrok.io\/system\/media_attachments\/files\/.+\/original\/attachment.jpg$/
-      end
-
-      it 'appends mastodon:scope element with visibility' do
-        status = Fabricate(:status, visibility: :public)
-
-        entry = OStatus::AtomSerializer.new.entry(status.stream_entry)
-
-        scope = entry.nodes.find { |node| node.name == 'mastodon:scope' }
-        expect(scope.text).to eq 'public'
-      end
-    end
-
-    context 'if status is not present' do
-      it 'appends content element saying status is deleted' do
-        status = Fabricate(:status)
-        status.destroy!
-
-        entry = OStatus::AtomSerializer.new.entry(status.stream_entry)
-
-        expect(entry.content.text).to eq 'Deleted status'
-      end
-
-      it 'appends title element saying the status is deleted' do
-        account = Fabricate(:account, username: 'username')
-        status = Fabricate(:status, account: account)
-        status.destroy!
-
-        entry = OStatus::AtomSerializer.new.entry(status.stream_entry)
-
-        expect(entry.title.text).to eq 'username deleted status'
-      end
-    end
-
-    context 'it is not root' do
-      let(:stream_entry) { Fabricate(:stream_entry) }
-      subject { OStatus::AtomSerializer.new.entry(stream_entry, false) }
-      include_examples 'not root'
-    end
-
-    context 'without root parameter' do
-      let(:stream_entry) { Fabricate(:stream_entry) }
-      subject { OStatus::AtomSerializer.new.entry(stream_entry) }
-      include_examples 'not root'
-    end
-
-    it 'returns entry element' do
-      stream_entry = Fabricate(:stream_entry)
-      entry = OStatus::AtomSerializer.new.entry(stream_entry)
-      expect(entry.name).to eq 'entry'
-    end
-
-    it 'appends id element with unique tag' do
-      status = Fabricate(:status, reblog_of_id: nil, created_at: '2000-01-01T00:00:00Z')
-
-      entry = OStatus::AtomSerializer.new.entry(status.stream_entry)
-
-      expect(entry.id.text).to eq "https://cb6e6126.ngrok.io/users/#{status.account.to_param}/statuses/#{status.id}"
-    end
-
-    it 'appends published element with created date' do
-      stream_entry = Fabricate(:stream_entry, created_at: '2000-01-01T00:00:00Z')
-      entry = OStatus::AtomSerializer.new.entry(stream_entry)
-      expect(entry.published.text).to eq '2000-01-01T00:00:00Z'
-    end
-
-    it 'appends updated element with updated date' do
-      stream_entry = Fabricate(:stream_entry, updated_at: '2000-01-01T00:00:00Z')
-      entry = OStatus::AtomSerializer.new.entry(stream_entry)
-      expect(entry.updated.text).to eq '2000-01-01T00:00:00Z'
-    end
-
-    it 'appends title element with status title' do
-      account = Fabricate(:account, username: 'username')
-      status = Fabricate(:status, account: account, reblog_of_id: nil)
-      entry = OStatus::AtomSerializer.new.entry(status.stream_entry)
-      expect(entry.title.text).to eq 'New status by username'
-    end
-
-    it 'appends activity:object-type element with object type' do
-      status = Fabricate(:status)
-      entry = OStatus::AtomSerializer.new.entry(status.stream_entry)
-      object_type = entry.nodes.find { |node| node.name == 'activity:object-type' }
-      expect(object_type.text).to eq OStatus::TagManager::TYPES[:note]
-    end
-
-    it 'appends activity:verb element with object type' do
-      status = Fabricate(:status)
-
-      entry = OStatus::AtomSerializer.new.entry(status.stream_entry)
-
-      object_type = entry.nodes.find { |node| node.name == 'activity:verb' }
-      expect(object_type.text).to eq OStatus::TagManager::VERBS[:post]
-    end
-
-    it 'appends activity:object element with target if present' do
-      reblogged = Fabricate(:status, created_at: '2000-01-01T00:00:00Z')
-      reblog = Fabricate(:status, reblog: reblogged)
-
-      entry = OStatus::AtomSerializer.new.entry(reblog.stream_entry)
-
-      object = entry.nodes.find { |node| node.name == 'activity:object' }
-      expect(object.id.text).to eq "https://cb6e6126.ngrok.io/users/#{reblogged.account.to_param}/statuses/#{reblogged.id}"
-    end
-
-    it 'does not append activity:object element if target is not present' do
-      status = Fabricate(:status, reblog_of_id: nil)
-      entry = OStatus::AtomSerializer.new.entry(status.stream_entry)
-      entry.nodes.each { |node| expect(node.name).not_to eq 'activity:object' }
-    end
-
-    it 'appends link element for an alternative' do
-      account = Fabricate(:account, username: 'username')
-      status = Fabricate(:status, account: account)
-
-      entry = OStatus::AtomSerializer.new.entry(status.stream_entry)
-
-      link = entry.nodes.find { |node| node.name == 'link' && node[:rel] == 'alternate' && node[:type] == 'text/html' }
-      expect(link[:type]).to eq 'text/html'
-      expect(link[:href]).to eq "https://cb6e6126.ngrok.io/@username/#{status.id}"
-    end
-
-    it 'appends link element for itself' do
-      account = Fabricate(:account, username: 'username')
-      status = Fabricate(:status, account: account)
-
-      entry = OStatus::AtomSerializer.new.entry(status.stream_entry)
-
-      link = entry.nodes.find { |node| node.name == 'link' && node[:rel] == 'self' }
-      expect(link[:type]).to eq 'application/atom+xml'
-      expect(link[:href]).to eq "https://cb6e6126.ngrok.io/users/username/updates/#{status.stream_entry.id}.atom"
-    end
-
-    it 'appends thr:in-reply-to element if threaded' do
-      in_reply_to_status = Fabricate(:status, created_at: '2000-01-01T00:00:00Z', reblog_of_id: nil)
-      reply_status = Fabricate(:status, in_reply_to_id: in_reply_to_status.id)
-
-      entry = OStatus::AtomSerializer.new.entry(reply_status.stream_entry)
-
-      in_reply_to = entry.nodes.find { |node| node.name == 'thr:in-reply-to' }
-      expect(in_reply_to[:ref]).to eq "https://cb6e6126.ngrok.io/users/#{in_reply_to_status.account.to_param}/statuses/#{in_reply_to_status.id}"
-    end
-
-    it 'does not append thr:in-reply-to element if not threaded' do
-      status = Fabricate(:status)
-      entry = OStatus::AtomSerializer.new.entry(status.stream_entry)
-      entry.nodes.each { |node| expect(node.name).not_to eq 'thr:in-reply-to' }
-    end
-
-    it 'appends ostatus:conversation if conversation id is present' do
-      status = Fabricate(:status)
-      status.conversation.update!(created_at: '2000-01-01T00:00:00Z')
-
-      entry = OStatus::AtomSerializer.new.entry(status.stream_entry)
-
-      conversation = entry.nodes.find { |node| node.name == 'ostatus:conversation' }
-      expect(conversation[:ref]).to eq "tag:cb6e6126.ngrok.io,2000-01-01:objectId=#{status.conversation_id}:objectType=Conversation"
-    end
-
-    it 'does not append ostatus:conversation if conversation id is not present' do
-      status = Fabricate.build(:status, conversation_id: nil)
-      status.save!(validate: false)
-
-      entry = OStatus::AtomSerializer.new.entry(status.stream_entry)
-
-      entry.nodes.each { |node| expect(node.name).not_to eq 'ostatus:conversation' }
-    end
-  end
-
-  describe '#feed' do
-    include_examples 'namespaces' do
-      def serialize
-        account = Fabricate(:account)
-        OStatus::AtomSerializer.new.feed(account, [])
-      end
-    end
-
-    it 'returns feed element' do
-      account = Fabricate(:account)
-      feed = OStatus::AtomSerializer.new.feed(account, [])
-      expect(feed.name).to eq 'feed'
-    end
-
-    it 'appends id element with account Atom URL' do
-      account = Fabricate(:account, username: 'username')
-      feed = OStatus::AtomSerializer.new.feed(account, [])
-      expect(feed.id.text).to eq 'https://cb6e6126.ngrok.io/users/username.atom'
-    end
-
-    it 'appends title element with account display name if present' do
-      account = Fabricate(:account, display_name: 'display name')
-      feed = OStatus::AtomSerializer.new.feed(account, [])
-      expect(feed.title.text).to eq 'display name'
-    end
-
-    it 'does not append title element with account username if account display name is not present' do
-      account = Fabricate(:account, display_name: '', username: 'username')
-      feed = OStatus::AtomSerializer.new.feed(account, [])
-      expect(feed.title.text).to eq 'username'
-    end
-
-    it 'appends subtitle element with account note' do
-      account = Fabricate(:account, note: 'note')
-      feed = OStatus::AtomSerializer.new.feed(account, [])
-      expect(feed.subtitle.text).to eq 'note'
-    end
-
-    it 'appends updated element with date account got updated' do
-      account = Fabricate(:account, updated_at: '2000-01-01T00:00:00Z')
-      feed = OStatus::AtomSerializer.new.feed(account, [])
-      expect(feed.updated.text).to eq '2000-01-01T00:00:00Z'
-    end
-
-    it 'appends logo element with full asset URL for original account avatar' do
-      account = Fabricate(:account, avatar: attachment_fixture('avatar.gif'))
-      feed = OStatus::AtomSerializer.new.feed(account, [])
-      expect(feed.logo.text).to match /^https:\/\/cb6e6126.ngrok.io\/system\/accounts\/avatars\/.+\/original\/avatar.gif/
-    end
-
-    it 'appends author element' do
-      account = Fabricate(:account, username: 'username')
-      feed = OStatus::AtomSerializer.new.feed(account, [])
-      expect(feed.author.id.text).to eq 'https://cb6e6126.ngrok.io/users/username'
-    end
-
-    it 'appends link element for an alternative' do
-      account = Fabricate(:account, username: 'username')
-
-      feed = OStatus::AtomSerializer.new.feed(account, [])
-
-      link = feed.nodes.find { |node| node.name == 'link' && node[:rel] == 'alternate' && node[:type] == 'text/html' }
-      expect(link[:type]).to eq 'text/html'
-      expect(link[:href]).to eq 'https://cb6e6126.ngrok.io/@username'
-    end
-
-    it 'appends link element for itself' do
-      account = Fabricate(:account, username: 'username')
-
-      feed = OStatus::AtomSerializer.new.feed(account, [])
-
-      link = feed.nodes.find { |node| node.name == 'link' && node[:rel] == 'self' }
-      expect(link[:type]).to eq 'application/atom+xml'
-      expect(link[:href]).to eq 'https://cb6e6126.ngrok.io/users/username.atom'
-    end
-
-    it 'appends link element for the next if it has 20 stream entries' do
-      account = Fabricate(:account, username: 'username')
-      stream_entry = Fabricate(:stream_entry)
-
-      feed = OStatus::AtomSerializer.new.feed(account, Array.new(20, stream_entry))
-
-      link = feed.nodes.find { |node| node.name == 'link' && node[:rel] == 'next' }
-      expect(link[:type]).to eq 'application/atom+xml'
-      expect(link[:href]).to eq "https://cb6e6126.ngrok.io/users/username.atom?max_id=#{stream_entry.id}"
-    end
-
-    it 'does not append link element for the next if it does not have 20 stream entries' do
-      account = Fabricate(:account, username: 'username')
-
-      feed = OStatus::AtomSerializer.new.feed(account, [])
-
-      feed.nodes.each do |node|
-        expect(node[:rel]).not_to eq 'next' if node.name == 'link'
-      end
-    end
-
-    it 'appends stream entries' do
-      account = Fabricate(:account, username: 'username')
-      status = Fabricate(:status, account: account)
-
-      feed = OStatus::AtomSerializer.new.feed(account, [status.stream_entry])
-
-      expect(feed.entry.title.text).to eq 'New status by username'
-    end
-  end
-
-  describe '#block_salmon' do
-    include_examples 'namespaces' do
-      def serialize
-        block = Fabricate(:block)
-        OStatus::AtomSerializer.new.block_salmon(block)
-      end
-    end
-
-    it 'returns entry element' do
-      block = Fabricate(:block)
-      block_salmon = OStatus::AtomSerializer.new.block_salmon(block)
-      expect(block_salmon.name).to eq 'entry'
-    end
-
-    it 'appends id element with unique tag' do
-      block = Fabricate(:block)
-
-      time_before = Time.zone.now
-      block_salmon = OStatus::AtomSerializer.new.block_salmon(block)
-      time_after = Time.zone.now
-
-      expect(block_salmon.id.text).to(
-        eq(OStatus::TagManager.instance.unique_tag(time_before.utc, block.id, 'Block'))
-          .or(eq(OStatus::TagManager.instance.unique_tag(time_after.utc, block.id, 'Block')))
-      )
-    end
-
-    it 'appends title element with description' do
-      account = Fabricate(:account, domain: nil, username: 'account')
-      target_account = Fabricate(:account, domain: 'remote', username: 'target_account')
-      block = Fabricate(:block, account: account, target_account: target_account)
-
-      block_salmon = OStatus::AtomSerializer.new.block_salmon(block)
-
-      expect(block_salmon.title.text).to eq 'account no longer wishes to interact with target_account@remote'
-    end
-
-    it 'appends author element with account' do
-      account = Fabricate(:account, domain: nil, username: 'account')
-      block = Fabricate(:block, account: account)
-
-      block_salmon = OStatus::AtomSerializer.new.block_salmon(block)
-
-      expect(block_salmon.author.id.text).to eq 'https://cb6e6126.ngrok.io/users/account'
-    end
-
-    it 'appends activity:object-type element with activity type' do
-      block = Fabricate(:block)
-
-      block_salmon = OStatus::AtomSerializer.new.block_salmon(block)
-
-      object_type = block_salmon.nodes.find { |node| node.name == 'activity:object-type' }
-      expect(object_type.text).to eq OStatus::TagManager::TYPES[:activity]
-    end
-
-    it 'appends activity:verb element with block' do
-      block = Fabricate(:block)
-
-      block_salmon = OStatus::AtomSerializer.new.block_salmon(block)
-
-      verb = block_salmon.nodes.find { |node| node.name == 'activity:verb' }
-      expect(verb.text).to eq OStatus::TagManager::VERBS[:block]
-    end
-
-    it 'appends activity:object element with target account' do
-      target_account = Fabricate(:account, domain: 'domain.test', uri: 'https://domain.test/id')
-      block = Fabricate(:block, target_account: target_account)
-
-      block_salmon = OStatus::AtomSerializer.new.block_salmon(block)
-
-      object = block_salmon.nodes.find { |node| node.name == 'activity:object' }
-      expect(object.id.text).to eq 'https://domain.test/id'
-    end
-  end
-
-  describe '#unblock_salmon' do
-    include_examples 'namespaces' do
-      def serialize
-        block = Fabricate(:block)
-        OStatus::AtomSerializer.new.unblock_salmon(block)
-      end
-    end
-
-    it 'returns entry element' do
-      block = Fabricate(:block)
-      unblock_salmon = OStatus::AtomSerializer.new.unblock_salmon(block)
-      expect(unblock_salmon.name).to eq 'entry'
-    end
-
-    it 'appends id element with unique tag' do
-      block = Fabricate(:block)
-
-      time_before = Time.zone.now
-      unblock_salmon = OStatus::AtomSerializer.new.unblock_salmon(block)
-      time_after = Time.zone.now
-
-      expect(unblock_salmon.id.text).to(
-        eq(OStatus::TagManager.instance.unique_tag(time_before.utc, block.id, 'Block'))
-          .or(eq(OStatus::TagManager.instance.unique_tag(time_after.utc, block.id, 'Block')))
-      )
-    end
-
-    it 'appends title element with description' do
-      account = Fabricate(:account, domain: nil, username: 'account')
-      target_account = Fabricate(:account, domain: 'remote', username: 'target_account')
-      block = Fabricate(:block, account: account, target_account: target_account)
-
-      unblock_salmon = OStatus::AtomSerializer.new.unblock_salmon(block)
-
-      expect(unblock_salmon.title.text).to eq 'account no longer blocks target_account@remote'
-    end
-
-    it 'appends author element with account' do
-      account = Fabricate(:account, domain: nil, username: 'account')
-      block = Fabricate(:block, account: account)
-
-      unblock_salmon = OStatus::AtomSerializer.new.unblock_salmon(block)
-
-      expect(unblock_salmon.author.id.text).to eq 'https://cb6e6126.ngrok.io/users/account'
-    end
-
-    it 'appends activity:object-type element with activity type' do
-      block = Fabricate(:block)
-
-      unblock_salmon = OStatus::AtomSerializer.new.unblock_salmon(block)
-
-      object_type = unblock_salmon.nodes.find { |node| node.name == 'activity:object-type' }
-      expect(object_type.text).to eq OStatus::TagManager::TYPES[:activity]
-    end
-
-    it 'appends activity:verb element with block' do
-      block = Fabricate(:block)
-
-      unblock_salmon = OStatus::AtomSerializer.new.unblock_salmon(block)
-
-      verb = unblock_salmon.nodes.find { |node| node.name == 'activity:verb' }
-      expect(verb.text).to eq OStatus::TagManager::VERBS[:unblock]
-    end
-
-    it 'appends activity:object element with target account' do
-      target_account = Fabricate(:account, domain: 'domain.test', uri: 'https://domain.test/id')
-      block = Fabricate(:block, target_account: target_account)
-
-      unblock_salmon = OStatus::AtomSerializer.new.unblock_salmon(block)
-
-      object = unblock_salmon.nodes.find { |node| node.name == 'activity:object' }
-      expect(object.id.text).to eq 'https://domain.test/id'
-    end
-  end
-
-  describe '#favourite_salmon' do
-    include_examples 'namespaces' do
-      def serialize
-        favourite = Fabricate(:favourite)
-        OStatus::AtomSerializer.new.favourite_salmon(favourite)
-      end
-    end
-
-    it 'returns entry element' do
-      favourite = Fabricate(:favourite)
-      favourite_salmon = OStatus::AtomSerializer.new.favourite_salmon(favourite)
-      expect(favourite_salmon.name).to eq 'entry'
-    end
-
-    it 'appends id element with unique tag' do
-      favourite = Fabricate(:favourite, created_at: '2000-01-01T00:00:00Z')
-      favourite_salmon = OStatus::AtomSerializer.new.favourite_salmon(favourite)
-      expect(favourite_salmon.id.text).to eq "tag:cb6e6126.ngrok.io,2000-01-01:objectId=#{favourite.id}:objectType=Favourite"
-    end
-
-    it 'appends author element with account' do
-      account = Fabricate(:account, domain: nil, username: 'username')
-      favourite = Fabricate(:favourite, account: account)
-
-      favourite_salmon = OStatus::AtomSerializer.new.favourite_salmon(favourite)
-
-      expect(favourite_salmon.author.id.text).to eq 'https://cb6e6126.ngrok.io/users/username'
-    end
-
-    it 'appends activity:object-type element with activity type' do
-      favourite = Fabricate(:favourite)
-
-      favourite_salmon = OStatus::AtomSerializer.new.favourite_salmon(favourite)
-
-      object_type = favourite_salmon.nodes.find { |node| node.name == 'activity:object-type' }
-      expect(object_type.text).to eq 'http://activitystrea.ms/schema/1.0/activity'
-    end
-
-    it 'appends activity:verb element with favorite' do
-      favourite = Fabricate(:favourite)
-
-      favourite_salmon = OStatus::AtomSerializer.new.favourite_salmon(favourite)
-
-      verb = favourite_salmon.nodes.find { |node| node.name == 'activity:verb' }
-      expect(verb.text).to eq OStatus::TagManager::VERBS[:favorite]
-    end
-
-    it 'appends activity:object element with status' do
-      status = Fabricate(:status, created_at: '2000-01-01T00:00:00Z')
-      favourite = Fabricate(:favourite, status: status)
-
-      favourite_salmon = OStatus::AtomSerializer.new.favourite_salmon(favourite)
-
-      object = favourite_salmon.nodes.find { |node| node.name == 'activity:object' }
-      expect(object.id.text).to eq "https://cb6e6126.ngrok.io/users/#{status.account.to_param}/statuses/#{status.id}"
-    end
-
-    it 'appends thr:in-reply-to element for status' do
-      status_account = Fabricate(:account, username: 'username')
-      status = Fabricate(:status, account: status_account, created_at: '2000-01-01T00:00:00Z')
-      favourite = Fabricate(:favourite, status: status)
-
-      favourite_salmon = OStatus::AtomSerializer.new.favourite_salmon(favourite)
-
-      in_reply_to = favourite_salmon.nodes.find { |node| node.name == 'thr:in-reply-to' }
-      expect(in_reply_to.ref).to eq "https://cb6e6126.ngrok.io/users/#{status.account.to_param}/statuses/#{status.id}"
-      expect(in_reply_to.href).to eq "https://cb6e6126.ngrok.io/@username/#{status.id}"
-    end
-
-    it 'includes description' do
-      account = Fabricate(:account, domain: nil, username: 'account')
-      status_account = Fabricate(:account, domain: 'remote', username: 'status_account')
-      status = Fabricate(:status, account: status_account)
-      favourite = Fabricate(:favourite, account: account, status: status)
-
-      favourite_salmon = OStatus::AtomSerializer.new.favourite_salmon(favourite)
-
-      expect(favourite_salmon.title.text).to eq 'account favourited a status by status_account@remote'
-      expect(favourite_salmon.content.text).to eq 'account favourited a status by status_account@remote'
-    end
-  end
-
-  describe '#unfavourite_salmon' do
-    include_examples 'namespaces' do
-      def serialize
-        favourite = Fabricate(:favourite)
-        OStatus::AtomSerializer.new.favourite_salmon(favourite)
-      end
-    end
-
-    it 'returns entry element' do
-      favourite = Fabricate(:favourite)
-      unfavourite_salmon = OStatus::AtomSerializer.new.unfavourite_salmon(favourite)
-      expect(unfavourite_salmon.name).to eq 'entry'
-    end
-
-    it 'appends id element with unique tag' do
-      favourite = Fabricate(:favourite)
-
-      time_before = Time.zone.now
-      unfavourite_salmon = OStatus::AtomSerializer.new.unfavourite_salmon(favourite)
-      time_after = Time.zone.now
-
-      expect(unfavourite_salmon.id.text).to(
-        eq(OStatus::TagManager.instance.unique_tag(time_before.utc, favourite.id, 'Favourite'))
-          .or(eq(OStatus::TagManager.instance.unique_tag(time_after.utc, favourite.id, 'Favourite')))
-      )
-    end
-
-    it 'appends author element with account' do
-      account = Fabricate(:account, domain: nil, username: 'username')
-      favourite = Fabricate(:favourite, account: account)
-
-      unfavourite_salmon = OStatus::AtomSerializer.new.unfavourite_salmon(favourite)
-
-      expect(unfavourite_salmon.author.id.text).to eq 'https://cb6e6126.ngrok.io/users/username'
-    end
-
-    it 'appends activity:object-type element with activity type' do
-      favourite = Fabricate(:favourite)
-
-      unfavourite_salmon = OStatus::AtomSerializer.new.unfavourite_salmon(favourite)
-
-      object_type = unfavourite_salmon.nodes.find { |node| node.name == 'activity:object-type' }
-      expect(object_type.text).to eq 'http://activitystrea.ms/schema/1.0/activity'
-    end
-
-    it 'appends activity:verb element with favorite' do
-      favourite = Fabricate(:favourite)
-
-      unfavourite_salmon = OStatus::AtomSerializer.new.unfavourite_salmon(favourite)
-
-      verb = unfavourite_salmon.nodes.find { |node| node.name == 'activity:verb' }
-      expect(verb.text).to eq OStatus::TagManager::VERBS[:unfavorite]
-    end
-
-    it 'appends activity:object element with status' do
-      status = Fabricate(:status, created_at: '2000-01-01T00:00:00Z')
-      favourite = Fabricate(:favourite, status: status)
-
-      unfavourite_salmon = OStatus::AtomSerializer.new.unfavourite_salmon(favourite)
-
-      object = unfavourite_salmon.nodes.find { |node| node.name == 'activity:object' }
-      expect(object.id.text).to eq "https://cb6e6126.ngrok.io/users/#{status.account.to_param}/statuses/#{status.id}"
-    end
-
-    it 'appends thr:in-reply-to element for status' do
-      status_account = Fabricate(:account, username: 'username')
-      status = Fabricate(:status, account: status_account, created_at: '2000-01-01T00:00:00Z')
-      favourite = Fabricate(:favourite, status: status)
-
-      unfavourite_salmon = OStatus::AtomSerializer.new.unfavourite_salmon(favourite)
-
-      in_reply_to = unfavourite_salmon.nodes.find { |node| node.name == 'thr:in-reply-to' }
-      expect(in_reply_to.ref).to eq "https://cb6e6126.ngrok.io/users/#{status.account.to_param}/statuses/#{status.id}"
-      expect(in_reply_to.href).to eq "https://cb6e6126.ngrok.io/@username/#{status.id}"
-    end
-
-    it 'includes description' do
-      account = Fabricate(:account, domain: nil, username: 'account')
-      status_account = Fabricate(:account, domain: 'remote', username: 'status_account')
-      status = Fabricate(:status, account: status_account)
-      favourite = Fabricate(:favourite, account: account, status: status)
-
-      unfavourite_salmon = OStatus::AtomSerializer.new.unfavourite_salmon(favourite)
-
-      expect(unfavourite_salmon.title.text).to eq 'account no longer favourites a status by status_account@remote'
-      expect(unfavourite_salmon.content.text).to eq 'account no longer favourites a status by status_account@remote'
-    end
-  end
-
-  describe '#follow_salmon' do
-    include_examples 'namespaces' do
-      def serialize
-        follow = Fabricate(:follow)
-        OStatus::AtomSerializer.new.follow_salmon(follow)
-      end
-    end
-
-    it 'returns entry element' do
-      follow = Fabricate(:follow)
-      follow_salmon = OStatus::AtomSerializer.new.follow_salmon(follow)
-      expect(follow_salmon.name).to eq 'entry'
-    end
-
-    it 'appends id element with unique tag' do
-      follow = Fabricate(:follow, created_at: '2000-01-01T00:00:00Z')
-      follow_salmon = OStatus::AtomSerializer.new.follow_salmon(follow)
-      expect(follow_salmon.id.text).to eq "tag:cb6e6126.ngrok.io,2000-01-01:objectId=#{follow.id}:objectType=Follow"
-    end
-
-    it 'appends author element with account' do
-      account = Fabricate(:account, domain: nil, username: 'username')
-      follow = Fabricate(:follow, account: account)
-
-      follow_salmon = OStatus::AtomSerializer.new.follow_salmon(follow)
-
-      expect(follow_salmon.author.id.text).to eq 'https://cb6e6126.ngrok.io/users/username'
-    end
-
-    it 'appends activity:object-type element with activity type' do
-      follow = Fabricate(:follow)
-
-      follow_salmon = OStatus::AtomSerializer.new.follow_salmon(follow)
-
-      object_type = follow_salmon.nodes.find { |node| node.name == 'activity:object-type' }
-      expect(object_type.text).to eq OStatus::TagManager::TYPES[:activity]
-    end
-
-    it 'appends activity:verb element with follow' do
-      follow = Fabricate(:follow)
-
-      follow_salmon = OStatus::AtomSerializer.new.follow_salmon(follow)
-
-      verb = follow_salmon.nodes.find { |node| node.name == 'activity:verb' }
-      expect(verb.text).to eq OStatus::TagManager::VERBS[:follow]
-    end
-
-    it 'appends activity:object element with target account' do
-      target_account = Fabricate(:account, domain: 'domain.test', uri: 'https://domain.test/id')
-      follow = Fabricate(:follow, target_account: target_account)
-
-      follow_salmon = OStatus::AtomSerializer.new.follow_salmon(follow)
-
-      object = follow_salmon.nodes.find { |node| node.name == 'activity:object' }
-      expect(object.id.text).to eq 'https://domain.test/id'
-    end
-
-    it 'includes description' do
-      account = Fabricate(:account, domain: nil, username: 'account')
-      target_account = Fabricate(:account, domain: 'remote', username: 'target_account')
-      follow = Fabricate(:follow, account: account, target_account: target_account)
-
-      follow_salmon = OStatus::AtomSerializer.new.follow_salmon(follow)
-
-      expect(follow_salmon.title.text).to eq 'account started following target_account@remote'
-      expect(follow_salmon.content.text).to eq 'account started following target_account@remote'
-    end
-  end
-
-  describe '#unfollow_salmon' do
-    include_examples 'namespaces' do
-      def serialize
-        follow = Fabricate(:follow)
-        follow.destroy!
-        OStatus::AtomSerializer.new.unfollow_salmon(follow)
-      end
-    end
-
-    it 'returns entry element' do
-      follow = Fabricate(:follow)
-      follow.destroy!
-
-      unfollow_salmon = OStatus::AtomSerializer.new.unfollow_salmon(follow)
-
-      expect(unfollow_salmon.name).to eq 'entry'
-    end
-
-    it 'appends id element with unique tag' do
-      follow = Fabricate(:follow)
-      follow.destroy!
-
-      time_before = Time.zone.now
-      unfollow_salmon = OStatus::AtomSerializer.new.unfollow_salmon(follow)
-      time_after = Time.zone.now
-
-      expect(unfollow_salmon.id.text).to(
-        eq(OStatus::TagManager.instance.unique_tag(time_before.utc, follow.id, 'Follow'))
-          .or(eq(OStatus::TagManager.instance.unique_tag(time_after.utc, follow.id, 'Follow')))
-      )
-    end
-
-    it 'appends title element with description' do
-      account = Fabricate(:account, domain: nil, username: 'account')
-      target_account = Fabricate(:account, domain: 'remote', username: 'target_account')
-      follow = Fabricate(:follow, account: account, target_account: target_account)
-      follow.destroy!
-
-      unfollow_salmon = OStatus::AtomSerializer.new.unfollow_salmon(follow)
-
-      expect(unfollow_salmon.title.text).to eq 'account is no longer following target_account@remote'
-    end
-
-    it 'appends content element with description' do
-      account = Fabricate(:account, domain: nil, username: 'account')
-      target_account = Fabricate(:account, domain: 'remote', username: 'target_account')
-      follow = Fabricate(:follow, account: account, target_account: target_account)
-      follow.destroy!
-
-      unfollow_salmon = OStatus::AtomSerializer.new.unfollow_salmon(follow)
-
-      expect(unfollow_salmon.content.text).to eq 'account is no longer following target_account@remote'
-    end
-
-    it 'appends author element with account' do
-      account = Fabricate(:account, domain: nil, username: 'username')
-      follow = Fabricate(:follow, account: account)
-      follow.destroy!
-
-      unfollow_salmon = OStatus::AtomSerializer.new.unfollow_salmon(follow)
-
-      expect(unfollow_salmon.author.id.text).to eq 'https://cb6e6126.ngrok.io/users/username'
-    end
-
-    it 'appends activity:object-type element with activity type' do
-      follow = Fabricate(:follow)
-      follow.destroy!
-
-      unfollow_salmon = OStatus::AtomSerializer.new.unfollow_salmon(follow)
-
-      object_type = unfollow_salmon.nodes.find { |node| node.name == 'activity:object-type' }
-      expect(object_type.text).to eq OStatus::TagManager::TYPES[:activity]
-    end
-
-    it 'appends activity:verb element with follow' do
-      follow = Fabricate(:follow)
-      follow.destroy!
-
-      unfollow_salmon = OStatus::AtomSerializer.new.unfollow_salmon(follow)
-
-      verb = unfollow_salmon.nodes.find { |node| node.name == 'activity:verb' }
-      expect(verb.text).to eq OStatus::TagManager::VERBS[:unfollow]
-    end
-
-    it 'appends activity:object element with target account' do
-      target_account = Fabricate(:account, domain: 'domain.test', uri: 'https://domain.test/id')
-      follow = Fabricate(:follow, target_account: target_account)
-      follow.destroy!
-
-      unfollow_salmon = OStatus::AtomSerializer.new.unfollow_salmon(follow)
-
-      object = unfollow_salmon.nodes.find { |node| node.name == 'activity:object' }
-      expect(object.id.text).to eq 'https://domain.test/id'
-    end
-  end
-
-  describe '#follow_request_salmon' do
-    include_examples 'namespaces' do
-      def serialize
-        follow_request = Fabricate(:follow_request)
-        OStatus::AtomSerializer.new.follow_request_salmon(follow_request)
-      end
-    end
-
-    context do
-      def serialize(follow_request)
-        OStatus::AtomSerializer.new.follow_request_salmon(follow_request)
-      end
-
-      it_behaves_like 'follow request salmon'
-
-      it 'appends id element with unique tag' do
-        follow_request = Fabricate(:follow_request, created_at: '2000-01-01T00:00:00Z')
-        follow_request_salmon = serialize(follow_request)
-        expect(follow_request_salmon.id.text).to eq "tag:cb6e6126.ngrok.io,2000-01-01:objectId=#{follow_request.id}:objectType=FollowRequest"
-      end
-
-      it 'appends title element with description' do
-        account = Fabricate(:account, domain: nil, username: 'account')
-        target_account = Fabricate(:account, domain: 'remote', username: 'target_account')
-        follow_request = Fabricate(:follow_request, account: account, target_account: target_account)
-        follow_request_salmon = serialize(follow_request)
-        expect(follow_request_salmon.title.text).to eq 'account requested to follow target_account@remote'
-      end
-    end
-  end
-
-  describe '#authorize_follow_request_salmon' do
-    include_examples 'namespaces' do
-      def serialize
-        follow_request = Fabricate(:follow_request)
-        OStatus::AtomSerializer.new.authorize_follow_request_salmon(follow_request)
-      end
-    end
-
-    it_behaves_like 'follow request salmon' do
-      def serialize(follow_request)
-        authorize_follow_request_salmon = OStatus::AtomSerializer.new.authorize_follow_request_salmon(follow_request)
-        authorize_follow_request_salmon.nodes.find { |node| node.name == 'activity:object' }
-      end
-    end
-
-    it 'appends id element with unique tag' do
-      follow_request = Fabricate(:follow_request)
-
-      time_before = Time.zone.now
-      authorize_follow_request_salmon = OStatus::AtomSerializer.new.authorize_follow_request_salmon(follow_request)
-      time_after = Time.zone.now
-
-      expect(authorize_follow_request_salmon.id.text).to(
-        eq(OStatus::TagManager.instance.unique_tag(time_before.utc, follow_request.id, 'FollowRequest'))
-          .or(eq(OStatus::TagManager.instance.unique_tag(time_after.utc, follow_request.id, 'FollowRequest')))
-      )
-    end
-
-    it 'appends title element with description' do
-      account = Fabricate(:account, domain: 'remote', username: 'account')
-      target_account = Fabricate(:account, domain: nil, username: 'target_account')
-      follow_request = Fabricate(:follow_request, account: account, target_account: target_account)
-
-      authorize_follow_request_salmon = OStatus::AtomSerializer.new.authorize_follow_request_salmon(follow_request)
-
-      expect(authorize_follow_request_salmon.title.text).to eq 'target_account authorizes follow request by account@remote'
-    end
-
-    it 'appends activity:object-type element with activity type' do
-      follow_request = Fabricate(:follow_request)
-
-      authorize_follow_request_salmon = OStatus::AtomSerializer.new.authorize_follow_request_salmon(follow_request)
-
-      object_type = authorize_follow_request_salmon.nodes.find { |node| node.name == 'activity:object-type' }
-      expect(object_type.text).to eq OStatus::TagManager::TYPES[:activity]
-    end
-
-    it 'appends activity:verb element with authorize' do
-      follow_request = Fabricate(:follow_request)
-
-      authorize_follow_request_salmon = OStatus::AtomSerializer.new.authorize_follow_request_salmon(follow_request)
-
-      verb = authorize_follow_request_salmon.nodes.find { |node| node.name == 'activity:verb' }
-      expect(verb.text).to eq OStatus::TagManager::VERBS[:authorize]
-    end
-  end
-
-  describe '#reject_follow_request_salmon' do
-    include_examples 'namespaces' do
-      def serialize
-        follow_request = Fabricate(:follow_request)
-        OStatus::AtomSerializer.new.reject_follow_request_salmon(follow_request)
-      end
-    end
-
-    it_behaves_like 'follow request salmon' do
-      def serialize(follow_request)
-        reject_follow_request_salmon = OStatus::AtomSerializer.new.reject_follow_request_salmon(follow_request)
-        reject_follow_request_salmon.nodes.find { |node| node.name == 'activity:object' }
-      end
-    end
-
-    it 'appends id element with unique tag' do
-      follow_request = Fabricate(:follow_request)
-
-      time_before = Time.zone.now
-      reject_follow_request_salmon = OStatus::AtomSerializer.new.reject_follow_request_salmon(follow_request)
-      time_after = Time.zone.now
-
-      expect(reject_follow_request_salmon.id.text).to(
-        eq(OStatus::TagManager.instance.unique_tag(time_before.utc, follow_request.id, 'FollowRequest'))
-          .or(OStatus::TagManager.instance.unique_tag(time_after.utc, follow_request.id, 'FollowRequest'))
-      )
-    end
-
-    it 'appends title element with description' do
-      account = Fabricate(:account, domain: 'remote', username: 'account')
-      target_account = Fabricate(:account, domain: nil, username: 'target_account')
-      follow_request = Fabricate(:follow_request, account: account, target_account: target_account)
-      reject_follow_request_salmon = OStatus::AtomSerializer.new.reject_follow_request_salmon(follow_request)
-      expect(reject_follow_request_salmon.title.text).to eq 'target_account rejects follow request by account@remote'
-    end
-
-    it 'appends activity:object-type element with activity type' do
-      follow_request = Fabricate(:follow_request)
-      reject_follow_request_salmon = OStatus::AtomSerializer.new.reject_follow_request_salmon(follow_request)
-      object_type = reject_follow_request_salmon.nodes.find { |node| node.name == 'activity:object-type' }
-      expect(object_type.text).to eq OStatus::TagManager::TYPES[:activity]
-    end
-
-    it 'appends activity:verb element with authorize' do
-      follow_request = Fabricate(:follow_request)
-      reject_follow_request_salmon = OStatus::AtomSerializer.new.reject_follow_request_salmon(follow_request)
-      verb = reject_follow_request_salmon.nodes.find { |node| node.name == 'activity:verb' }
-      expect(verb.text).to eq OStatus::TagManager::VERBS[:reject]
-    end
-  end
-
-  describe '#object' do
-    include_examples 'status attributes' do
-      def serialize(status)
-        OStatus::AtomSerializer.new.object(status)
-      end
-    end
-
-    it 'returns activity:object element' do
-      status = Fabricate(:status)
-      object = OStatus::AtomSerializer.new.object(status)
-      expect(object.name).to eq 'activity:object'
-    end
-
-    it 'appends id element with URL for status' do
-      status = Fabricate(:status, created_at: '2000-01-01T00:00:00Z')
-      object = OStatus::AtomSerializer.new.object(status)
-      expect(object.id.text).to eq "https://cb6e6126.ngrok.io/users/#{status.account.to_param}/statuses/#{status.id}"
-    end
-
-    it 'appends published element with created date' do
-      status = Fabricate(:status, created_at: '2000-01-01T00:00:00Z')
-      object = OStatus::AtomSerializer.new.object(status)
-      expect(object.published.text).to eq '2000-01-01T00:00:00Z'
-    end
-
-    it 'appends updated element with updated date' do
-      status = Fabricate(:status)
-      status.updated_at = '2000-01-01T00:00:00Z'
-      object = OStatus::AtomSerializer.new.object(status)
-      expect(object.updated.text).to eq '2000-01-01T00:00:00Z'
-    end
-
-    it 'appends title element with title' do
-      account = Fabricate(:account, username: 'username')
-      status = Fabricate(:status, account: account)
-
-      object = OStatus::AtomSerializer.new.object(status)
-
-      expect(object.title.text).to eq 'New status by username'
-    end
-
-    it 'appends author element with account' do
-      account = Fabricate(:account, username: 'username')
-      status = Fabricate(:status, account: account)
-
-      entry = OStatus::AtomSerializer.new.object(status)
-
-      expect(entry.author.id.text).to eq 'https://cb6e6126.ngrok.io/users/username'
-    end
-
-    it 'appends activity:object-type element with object type' do
-      status = Fabricate(:status)
-
-      entry = OStatus::AtomSerializer.new.object(status)
-
-      object_type = entry.nodes.find { |node| node.name == 'activity:object-type' }
-      expect(object_type.text).to eq OStatus::TagManager::TYPES[:note]
-    end
-
-    it 'appends activity:verb element with verb' do
-      status = Fabricate(:status)
-
-      entry = OStatus::AtomSerializer.new.object(status)
-
-      object_type = entry.nodes.find { |node| node.name == 'activity:verb' }
-      expect(object_type.text).to eq OStatus::TagManager::VERBS[:post]
-    end
-
-    it 'appends link element for an alternative' do
-      account = Fabricate(:account, username: 'username')
-      status = Fabricate(:status, account: account)
-
-      entry = OStatus::AtomSerializer.new.object(status)
-
-      link = entry.nodes.find { |node| node.name == 'link' && node[:rel] == 'alternate' && node[:type] == 'text/html' }
-      expect(link[:type]).to eq 'text/html'
-      expect(link[:href]).to eq "https://cb6e6126.ngrok.io/@username/#{status.id}"
-    end
-
-    it 'appends thr:in-reply-to element if it is a reply and thread is not nil' do
-      account = Fabricate(:account, username: 'username')
-      thread = Fabricate(:status, account: account, created_at: '2000-01-01T00:00:00Z')
-      reply = Fabricate(:status, thread: thread)
-
-      entry = OStatus::AtomSerializer.new.object(reply)
-
-      in_reply_to = entry.nodes.find { |node| node.name == 'thr:in-reply-to' }
-      expect(in_reply_to.ref).to eq "https://cb6e6126.ngrok.io/users/#{thread.account.to_param}/statuses/#{thread.id}"
-      expect(in_reply_to.href).to eq "https://cb6e6126.ngrok.io/@username/#{thread.id}"
-    end
-
-    it 'does not append thr:in-reply-to element if thread is nil' do
-      status = Fabricate(:status, thread: nil)
-      entry = OStatus::AtomSerializer.new.object(status)
-      entry.nodes.each { |node| expect(node.name).not_to eq 'thr:in-reply-to' }
-    end
-
-    it 'does not append ostatus:conversation element if conversation_id is nil' do
-      status = Fabricate.build(:status, conversation_id: nil)
-      status.save!(validate: false)
-
-      entry = OStatus::AtomSerializer.new.object(status)
-
-      entry.nodes.each { |node| expect(node.name).not_to eq 'ostatus:conversation' }
-    end
-
-    it 'appends ostatus:conversation element if conversation_id is not nil' do
-      status = Fabricate(:status)
-      status.conversation.update!(created_at: '2000-01-01T00:00:00Z')
-
-      entry = OStatus::AtomSerializer.new.object(status)
-
-      conversation = entry.nodes.find { |node| node.name == 'ostatus:conversation' }
-      expect(conversation[:ref]).to eq "tag:cb6e6126.ngrok.io,2000-01-01:objectId=#{status.conversation.id}:objectType=Conversation"
-    end
-  end
-end
diff --git a/spec/lib/status_finder_spec.rb b/spec/lib/status_finder_spec.rb
index 6b4ee434f..61483f4bf 100644
--- a/spec/lib/status_finder_spec.rb
+++ b/spec/lib/status_finder_spec.rb
@@ -25,15 +25,6 @@ describe StatusFinder do
       end
     end
 
-    context 'with a stream entry url' do
-      let(:stream_entry) { Fabricate(:stream_entry) }
-      let(:url) { account_stream_entry_url(stream_entry.account, stream_entry) }
-
-      it 'finds the stream entry' do
-        expect(subject.status).to eq(stream_entry.status)
-      end
-    end
-
     context 'with a remote url even if id exists on local' do
       let(:status) { Fabricate(:status) }
       let(:url) { "https://example.com/users/test/statuses/#{status.id}" }
diff --git a/spec/lib/tag_manager_spec.rb b/spec/lib/tag_manager_spec.rb
index 3a804ac0f..e9a7aa934 100644
--- a/spec/lib/tag_manager_spec.rb
+++ b/spec/lib/tag_manager_spec.rb
@@ -119,46 +119,4 @@ RSpec.describe TagManager do
       expect(TagManager.instance.same_acct?('username', 'incorrect@Cb6E6126.nGrOk.Io')).to eq false
     end
   end
-
-  describe '#url_for' do
-    let(:alice) { Fabricate(:account, username: 'alice') }
-
-    subject { TagManager.instance.url_for(target) }
-
-    context 'activity object' do
-      let(:target) { Fabricate(:status, account: alice, reblog: Fabricate(:status)).stream_entry }
-
-      it 'returns the unique tag for status' do
-        expect(target.object_type).to eq :activity
-        is_expected.to eq "https://cb6e6126.ngrok.io/@alice/#{target.id}"
-      end
-    end
-
-    context 'comment object' do
-      let(:target) { Fabricate(:status, account: alice, reply: true) }
-
-      it 'returns the unique tag for status' do
-        expect(target.object_type).to eq :comment
-        is_expected.to eq "https://cb6e6126.ngrok.io/@alice/#{target.id}"
-      end
-    end
-
-    context 'note object' do
-      let(:target) { Fabricate(:status, account: alice, reply: false, thread: nil) }
-
-      it 'returns the unique tag for status' do
-        expect(target.object_type).to eq :note
-        is_expected.to eq "https://cb6e6126.ngrok.io/@alice/#{target.id}"
-      end
-    end
-
-    context 'person object' do
-      let(:target) { alice }
-
-      it 'returns the URL for account' do
-        expect(target.object_type).to eq :person
-        is_expected.to eq 'https://cb6e6126.ngrok.io/@alice'
-      end
-    end
-  end
 end
diff --git a/spec/models/concerns/streamable_spec.rb b/spec/models/concerns/streamable_spec.rb
deleted file mode 100644
index b5f2d5192..000000000
--- a/spec/models/concerns/streamable_spec.rb
+++ /dev/null
@@ -1,63 +0,0 @@
-# frozen_string_literal: true
-
-require 'rails_helper'
-
-RSpec.describe Streamable do
-  class Parent
-    def title; end
-
-    def target; end
-
-    def thread; end
-
-    def self.has_one(*); end
-
-    def self.after_create; end
-  end
-
-  class Child < Parent
-    include Streamable
-  end
-
-  child = Child.new
-
-  describe '#title' do
-    it 'calls Parent#title' do
-      expect_any_instance_of(Parent).to receive(:title)
-      child.title
-    end
-  end
-
-  describe '#content' do
-    it 'calls #title' do
-      expect_any_instance_of(Parent).to receive(:title)
-      child.content
-    end
-  end
-
-  describe '#target' do
-    it 'calls Parent#target' do
-      expect_any_instance_of(Parent).to receive(:target)
-      child.target
-    end
-  end
-
-  describe '#object_type' do
-    it 'returns :activity' do
-      expect(child.object_type).to eq :activity
-    end
-  end
-
-  describe '#thread' do
-    it 'calls Parent#thread' do
-      expect_any_instance_of(Parent).to receive(:thread)
-      child.thread
-    end
-  end
-
-  describe '#hidden?' do
-    it 'returns false' do
-      expect(child.hidden?).to be false
-    end
-  end
-end
diff --git a/spec/models/remote_profile_spec.rb b/spec/models/remote_profile_spec.rb
deleted file mode 100644
index da5048f0a..000000000
--- a/spec/models/remote_profile_spec.rb
+++ /dev/null
@@ -1,143 +0,0 @@
-# frozen_string_literal: true
-
-require 'rails_helper'
-
-RSpec.describe RemoteProfile do
-  let(:remote_profile) { RemoteProfile.new(body) }
-  let(:body) do
-    <<-XML
-      <feed xmlns="http://www.w3.org/2005/Atom">
-      <author>John</author>
-    XML
-  end
-
-  describe '.initialize' do
-    it 'calls Nokogiri::XML.parse' do
-      expect(Nokogiri::XML).to receive(:parse).with(body, nil, 'utf-8')
-      RemoteProfile.new(body)
-    end
-
-    it 'sets document' do
-      remote_profile = RemoteProfile.new(body)
-      expect(remote_profile).not_to be nil
-    end
-  end
-
-  describe '#root' do
-    let(:document) { remote_profile.document }
-
-    it 'callse document.at_xpath' do
-      expect(document).to receive(:at_xpath).with(
-        '/atom:feed|/atom:entry',
-        atom: OStatus::TagManager::XMLNS
-      )
-
-      remote_profile.root
-    end
-  end
-
-  describe '#author' do
-    let(:root) { remote_profile.root }
-
-    it 'calls root.at_xpath' do
-      expect(root).to receive(:at_xpath).with(
-        './atom:author|./dfrn:owner',
-        atom: OStatus::TagManager::XMLNS,
-        dfrn: OStatus::TagManager::DFRN_XMLNS
-      )
-
-      remote_profile.author
-    end
-  end
-
-  describe '#hub_link' do
-    let(:root) { remote_profile.root }
-
-    it 'calls #link_href_from_xml' do
-      expect(remote_profile).to receive(:link_href_from_xml).with(root, 'hub')
-      remote_profile.hub_link
-    end
-  end
-
-  describe '#display_name' do
-    let(:author) { remote_profile.author }
-
-    it 'calls author.at_xpath.content' do
-      expect(author).to receive_message_chain(:at_xpath, :content).with(
-        './poco:displayName',
-        poco: OStatus::TagManager::POCO_XMLNS
-      ).with(no_args)
-
-      remote_profile.display_name
-    end
-  end
-
-  describe '#note' do
-    let(:author) { remote_profile.author }
-
-    it 'calls author.at_xpath.content' do
-      expect(author).to receive_message_chain(:at_xpath, :content).with(
-        './atom:summary|./poco:note',
-        atom: OStatus::TagManager::XMLNS,
-        poco: OStatus::TagManager::POCO_XMLNS
-      ).with(no_args)
-
-      remote_profile.note
-    end
-  end
-
-  describe '#scope' do
-    let(:author) { remote_profile.author }
-
-    it 'calls author.at_xpath.content' do
-      expect(author).to receive_message_chain(:at_xpath, :content).with(
-        './mastodon:scope',
-        mastodon: OStatus::TagManager::MTDN_XMLNS
-      ).with(no_args)
-
-      remote_profile.scope
-    end
-  end
-
-  describe '#avatar' do
-    let(:author) { remote_profile.author }
-
-    it 'calls #link_href_from_xml' do
-      expect(remote_profile).to receive(:link_href_from_xml).with(author, 'avatar')
-      remote_profile.avatar
-    end
-  end
-
-  describe '#header' do
-    let(:author) { remote_profile.author }
-
-    it 'calls #link_href_from_xml' do
-      expect(remote_profile).to receive(:link_href_from_xml).with(author, 'header')
-      remote_profile.header
-    end
-  end
-
-  describe '#locked?' do
-    before do
-      allow(remote_profile).to receive(:scope).and_return(scope)
-    end
-
-    subject { remote_profile.locked? }
-
-    context 'scope is private' do
-      let(:scope) { 'private' }
-
-      it 'returns true' do
-        is_expected.to be true
-      end
-    end
-
-    context 'scope is not private' do
-      let(:scope) { 'public' }
-
-      it 'returns false' do
-        is_expected.to be false
-      end
-    end
-  end
-end
diff --git a/spec/models/stream_entry_spec.rb b/spec/models/stream_entry_spec.rb
deleted file mode 100644
index 8f8bfbd58..000000000
--- a/spec/models/stream_entry_spec.rb
+++ /dev/null
@@ -1,192 +0,0 @@
-require 'rails_helper'
-
-RSpec.describe StreamEntry, type: :model do
-  let(:alice)     { Fabricate(:account, username: 'alice') }
-  let(:bob)       { Fabricate(:account, username: 'bob') }
-  let(:status)    { Fabricate(:status, account: alice) }
-  let(:reblog)    { Fabricate(:status, account: bob, reblog: status) }
-  let(:reply)     { Fabricate(:status, account: bob, thread: status) }
-  let(:stream_entry) { Fabricate(:stream_entry, activity: activity) }
-  let(:activity)     { reblog }
-
-  describe '#object_type' do
-    before do
-      allow(stream_entry).to receive(:orphaned?).and_return(orphaned)
-      allow(stream_entry).to receive(:targeted?).and_return(targeted)
-    end
-
-    subject { stream_entry.object_type }
-
-    context 'orphaned? is true' do
-      let(:orphaned) { true }
-      let(:targeted) { false }
-
-      it 'returns :activity' do
-        is_expected.to be :activity
-      end
-    end
-
-    context 'targeted? is true' do
-      let(:orphaned) { false }
-      let(:targeted) { true }
-
-      it 'returns :activity' do
-        is_expected.to be :activity
-      end
-    end
-
-    context 'orphaned? and targeted? are false' do
-      let(:orphaned) { false }
-      let(:targeted) { false }
-
-      context 'activity is reblog' do
-        let(:activity) { reblog }
-
-        it 'returns :note' do
-          is_expected.to be :note
-        end
-      end
-
-      context 'activity is reply' do
-        let(:activity) { reply }
-
-        it 'returns :comment' do
-          is_expected.to be :comment
-        end
-      end
-    end
-  end
-
-  describe '#verb' do
-    before do
-      allow(stream_entry).to receive(:orphaned?).and_return(orphaned)
-    end
-
-    subject { stream_entry.verb }
-
-    context 'orphaned? is true' do
-      let(:orphaned) { true }
-
-      it 'returns :delete' do
-        is_expected.to be :delete
-      end
-    end
-
-    context 'orphaned? is false' do
-      let(:orphaned) { false }
-
-      context 'activity is reblog' do
-        let(:activity) { reblog }
-
-        it 'returns :share' do
-          is_expected.to be :share
-        end
-      end
-
-      context 'activity is reply' do
-        let(:activity) { reply }
-
-        it 'returns :post' do
-          is_expected.to be :post
-        end
-      end
-    end
-  end
-
-  describe '#mentions' do
-    before do
-      allow(stream_entry).to receive(:orphaned?).and_return(orphaned)
-    end
-
-    subject { stream_entry.mentions }
-
-    context 'orphaned? is true' do
-      let(:orphaned) { true }
-
-      it 'returns []' do
-        is_expected.to eq []
-      end
-    end
-
-    context 'orphaned? is false' do
-      before do
-        reblog.mentions << Fabricate(:mention, account: alice)
-        reblog.mentions << Fabricate(:mention, account: bob)
-      end
-
-      let(:orphaned) { false }
-
-      it 'returns [Account] includes alice and bob' do
-        is_expected.to eq [alice, bob]
-      end
-    end
-  end
-
-  describe '#targeted?' do
-    it 'returns true for a reblog' do
-      expect(reblog.stream_entry.targeted?).to be true
-    end
-
-    it 'returns false otherwise' do
-      expect(status.stream_entry.targeted?).to be false
-    end
-  end
-
-  describe '#threaded?' do
-    it 'returns true for a reply' do
-      expect(reply.stream_entry.threaded?).to be true
-    end
-
-    it 'returns false otherwise' do
-      expect(status.stream_entry.threaded?).to be false
-    end
-  end
-
-  describe 'delegated methods' do
-    context 'with a nil status' do
-      subject { described_class.new(status: nil) }
-
-      it 'returns nil for target' do
-        expect(subject.target).to be_nil
-      end
-
-      it 'returns nil for title' do
-        expect(subject.title).to be_nil
-      end
-
-      it 'returns nil for content' do
-        expect(subject.content).to be_nil
-      end
-
-      it 'returns nil for thread' do
-        expect(subject.thread).to be_nil
-      end
-    end
-
-    context 'with a real status' do
-      let(:original) { Fabricate(:status, text: 'Test status') }
-      let(:status) { Fabricate(:status, reblog: original, thread: original) }
-      subject { described_class.new(status: status) }
-
-      it 'delegates target' do
-        expect(status.target).not_to be_nil
-        expect(subject.target).to eq(status.target)
-      end
-
-      it 'delegates title' do
-        expect(status.title).not_to be_nil
-        expect(subject.title).to eq(status.title)
-      end
-
-      it 'delegates content' do
-        expect(status.content).not_to be_nil
-        expect(subject.content).to eq(status.content)
-      end
-
-      it 'delegates thread' do
-        expect(status.thread).not_to be_nil
-        expect(subject.thread).to eq(status.thread)
-      end
-    end
-  end
-end
diff --git a/spec/services/process_mentions_service_spec.rb b/spec/services/process_mentions_service_spec.rb
index 35a804f2b..b1abd79b0 100644
--- a/spec/services/process_mentions_service_spec.rb
+++ b/spec/services/process_mentions_service_spec.rb
@@ -15,8 +15,8 @@ RSpec.describe ProcessMentionsService, type: :service do
       subject.call(status)
     end
 
-    it 'creates a mention' do
-      expect(remote_user.mentions.where(status: status).count).to eq 1
+    it 'does not create a mention' do
+      expect(remote_user.mentions.where(status: status).count).to eq 0
     end
   end
 
diff --git a/spec/services/suspend_account_service_spec.rb b/spec/services/suspend_account_service_spec.rb
index 6f45762aa..896ac17a3 100644
--- a/spec/services/suspend_account_service_spec.rb
+++ b/spec/services/suspend_account_service_spec.rb
@@ -27,14 +27,13 @@ RSpec.describe SuspendAccountService, type: :service do
         [
           account.statuses,
           account.media_attachments,
-          account.stream_entries,
           account.notifications,
           account.favourites,
           account.active_relationships,
           account.passive_relationships,
           account.subscriptions
         ].map(&:count)
-      }.from([1, 1, 1, 1, 1, 1, 1, 1]).to([0, 0, 0, 0, 0, 0, 0, 0])
+      }.from([1, 1, 1, 1, 1, 1, 1]).to([0, 0, 0, 0, 0, 0, 0])
     end
 
     it 'sends a delete actor activity to all known inboxes' do
@@ -70,14 +69,13 @@ RSpec.describe SuspendAccountService, type: :service do
         [
           remote_bob.statuses,
           remote_bob.media_attachments,
-          remote_bob.stream_entries,
           remote_bob.notifications,
           remote_bob.favourites,
           remote_bob.active_relationships,
           remote_bob.passive_relationships,
           remote_bob.subscriptions
         ].map(&:count)
-      }.from([1, 1, 1, 1, 1, 1, 1, 1]).to([0, 0, 0, 0, 0, 0, 0, 0])
+      }.from([1, 1, 1, 1, 1, 1, 1]).to([0, 0, 0, 0, 0, 0, 0])
     end
 
     it 'sends a reject follow to follwer inboxes' do
diff --git a/spec/views/statuses/show.html.haml_spec.rb b/spec/views/statuses/show.html.haml_spec.rb
new file mode 100644
index 000000000..dbda3b665
--- /dev/null
+++ b/spec/views/statuses/show.html.haml_spec.rb
@@ -0,0 +1,81 @@
+# frozen_string_literal: true
+
+require 'rails_helper'
+
+describe 'statuses/show.html.haml', without_verify_partial_doubles: true do
+  before do
+    double(:api_oembed_url => '')
+    allow(view).to receive(:show_landing_strip?).and_return(true)
+    allow(view).to receive(:site_title).and_return('example site')
+    allow(view).to receive(:site_hostname).and_return('example.com')
+    allow(view).to receive(:full_asset_url).and_return('//asset.host/image.svg')
+    allow(view).to receive(:local_time)
+    allow(view).to receive(:local_time_ago)
+    allow(view).to receive(:current_account).and_return(nil)
+    assign(:instance_presenter, InstancePresenter.new)
+  end
+
+  it 'has valid author h-card and basic data for a detailed_status' do
+    alice  =  Fabricate(:account, username: 'alice', display_name: 'Alice')
+    bob    =  Fabricate(:account, username: 'bob', display_name: 'Bob')
+    status =  Fabricate(:status, account: alice, text: 'Hello World')
+    reply  =  Fabricate(:status, account: bob, thread: status, text: 'Hello Alice')
+
+    assign(:status, status)
+    assign(:account, alice)
+    assign(:descendant_threads, [])
+
+    render
+
+    mf2 = Microformats.parse(rendered)
+
+    expect(mf2.entry.url.to_s).not_to be_empty
+    expect(mf2.entry.author.name.to_s).to eq alice.display_name
+    expect(mf2.entry.author.url.to_s).not_to be_empty
+  end
+
+  it 'has valid h-cites for p-in-reply-to and p-comment' do
+    alice   =  Fabricate(:account, username: 'alice', display_name: 'Alice')
+    bob     =  Fabricate(:account, username: 'bob', display_name: 'Bob')
+    carl    =  Fabricate(:account, username: 'carl', display_name: 'Carl')
+    status  =  Fabricate(:status, account: alice, text: 'Hello World')
+    reply   =  Fabricate(:status, account: bob, thread: status, text: 'Hello Alice')
+    comment =  Fabricate(:status, account: carl, thread: reply, text: 'Hello Bob')
+
+    assign(:status, reply)
+    assign(:account, alice)
+    assign(:ancestors, reply.ancestors(1, bob))
+    assign(:descendant_threads, [{ statuses: reply.descendants(1) }])
+
+    render
+
+    mf2 = Microformats.parse(rendered)
+
+    expect(mf2.entry.url.to_s).not_to be_empty
+    expect(mf2.entry.comment.url.to_s).not_to be_empty
+    expect(mf2.entry.comment.author.name.to_s).to eq carl.display_name
+    expect(mf2.entry.comment.author.url.to_s).not_to be_empty
+
+    expect(mf2.entry.in_reply_to.url.to_s).not_to be_empty
+    expect(mf2.entry.in_reply_to.author.name.to_s).to eq alice.display_name
+    expect(mf2.entry.in_reply_to.author.url.to_s).not_to be_empty
+  end
+
+  it 'has valid opengraph tags' do
+    alice   =  Fabricate(:account, username: 'alice', display_name: 'Alice')
+    status  =  Fabricate(:status, account: alice, text: 'Hello World')
+
+    assign(:status, status)
+    assign(:account, alice)
+    assign(:descendant_threads, [])
+
+    render
+
+    header_tags = view.content_for(:header_tags)
+
+    expect(header_tags).to match(%r{<meta content=".+" property="og:title" />})
+    expect(header_tags).to match(%r{<meta content="article" property="og:type" />})
+    expect(header_tags).to match(%r{<meta content=".+" property="og:image" />})
+    expect(header_tags).to match(%r{<meta content="http://.+" property="og:url" />})
+  end
+end
diff --git a/spec/views/stream_entries/show.html.haml_spec.rb b/spec/views/stream_entries/show.html.haml_spec.rb
deleted file mode 100644
index 93f0adb99..000000000
--- a/spec/views/stream_entries/show.html.haml_spec.rb
+++ /dev/null
@@ -1,88 +0,0 @@
-# frozen_string_literal: true
-
-require 'rails_helper'
-
-describe 'stream_entries/show.html.haml', without_verify_partial_doubles: true do
-  before do
-    double(:api_oembed_url => '')
-    double(:account_stream_entry_url => '')
-    allow(view).to receive(:show_landing_strip?).and_return(true)
-    allow(view).to receive(:site_title).and_return('example site')
-    allow(view).to receive(:site_hostname).and_return('example.com')
-    allow(view).to receive(:full_asset_url).and_return('//asset.host/image.svg')
-    allow(view).to receive(:local_time)
-    allow(view).to receive(:local_time_ago)
-    allow(view).to receive(:current_account).and_return(nil)
-    assign(:instance_presenter, InstancePresenter.new)
-  end
-
-  it 'has valid author h-card and basic data for a detailed_status' do
-    alice  =  Fabricate(:account, username: 'alice', display_name: 'Alice')
-    bob    =  Fabricate(:account, username: 'bob', display_name: 'Bob')
-    status =  Fabricate(:status, account: alice, text: 'Hello World')
-    reply  =  Fabricate(:status, account: bob, thread: status, text: 'Hello Alice')
-
-    assign(:status, status)
-    assign(:stream_entry, status.stream_entry)
-    assign(:account, alice)
-    assign(:type, status.stream_entry.activity_type.downcase)
-    assign(:descendant_threads, [])
-
-    render
-
-    mf2 = Microformats.parse(rendered)
-
-    expect(mf2.entry.url.to_s).not_to be_empty
-    expect(mf2.entry.author.name.to_s).to eq alice.display_name
-    expect(mf2.entry.author.url.to_s).not_to be_empty
-  end
-
-  it 'has valid h-cites for p-in-reply-to and p-comment' do
-    alice   =  Fabricate(:account, username: 'alice', display_name: 'Alice')
-    bob     =  Fabricate(:account, username: 'bob', display_name: 'Bob')
-    carl    =  Fabricate(:account, username: 'carl', display_name: 'Carl')
-    status  =  Fabricate(:status, account: alice, text: 'Hello World')
-    reply   =  Fabricate(:status, account: bob, thread: status, text: 'Hello Alice')
-    comment =  Fabricate(:status, account: carl, thread: reply, text: 'Hello Bob')
-
-    assign(:status, reply)
-    assign(:stream_entry, reply.stream_entry)
-    assign(:account, alice)
-    assign(:type, reply.stream_entry.activity_type.downcase)
-    assign(:ancestors, reply.stream_entry.activity.ancestors(1, bob))
-    assign(:descendant_threads, [{ statuses: reply.stream_entry.activity.descendants(1) }])
-
-    render
-
-    mf2 = Microformats.parse(rendered)
-
-    expect(mf2.entry.url.to_s).not_to be_empty
-    expect(mf2.entry.comment.url.to_s).not_to be_empty
-    expect(mf2.entry.comment.author.name.to_s).to eq carl.display_name
-    expect(mf2.entry.comment.author.url.to_s).not_to be_empty
-
-    expect(mf2.entry.in_reply_to.url.to_s).not_to be_empty
-    expect(mf2.entry.in_reply_to.author.name.to_s).to eq alice.display_name
-    expect(mf2.entry.in_reply_to.author.url.to_s).not_to be_empty
-  end
-
-  it 'has valid opengraph tags' do
-    alice   =  Fabricate(:account, username: 'alice', display_name: 'Alice')
-    status  =  Fabricate(:status, account: alice, text: 'Hello World')
-
-    assign(:status, status)
-    assign(:stream_entry, status.stream_entry)
-    assign(:account, alice)
-    assign(:type, status.stream_entry.activity_type.downcase)
-    assign(:descendant_threads, [])
-
-    render
-
-    header_tags = view.content_for(:header_tags)
-
-    expect(header_tags).to match(%r{<meta content=".+" property="og:title" />})
-    expect(header_tags).to match(%r{<meta content="article" property="og:type" />})
-    expect(header_tags).to match(%r{<meta content=".+" property="og:image" />})
-    expect(header_tags).to match(%r{<meta content="http://.+" property="og:url" />})
-  end
-end
-- 
cgit 


From 63c7fe8e4892b22e80c015bf0ecb04496318623b Mon Sep 17 00:00:00 2001
From: Eugen Rochko <eugen@zeonfederated.com>
Date: Mon, 8 Jul 2019 12:03:45 +0200
Subject: Refactor controllers for statuses, accounts, and more (#11249)

---
 app/controllers/about_controller.rb                |  16 +-
 app/controllers/accounts_controller.rb             |  15 +-
 .../activitypub/collections_controller.rb          |  16 +-
 app/controllers/activitypub/inboxes_controller.rb  |   7 +-
 app/controllers/activitypub/outboxes_controller.rb |   6 +-
 app/controllers/activitypub/replies_controller.rb  |  68 +++++++++
 app/controllers/api/proofs_controller.rb           |  17 +--
 app/controllers/application_controller.rb          |   4 -
 .../concerns/account_controller_concern.rb         |  34 +----
 app/controllers/concerns/account_owned_concern.rb  |  33 +++++
 .../concerns/status_controller_concern.rb          |  87 +++++++++++
 app/controllers/custom_css_controller.rb           |   1 +
 app/controllers/emojis_controller.rb               |   5 +-
 app/controllers/follower_accounts_controller.rb    |   2 +-
 app/controllers/following_accounts_controller.rb   |   2 +-
 app/controllers/home_controller.rb                 |   2 +-
 app/controllers/intents_controller.rb              |   1 +
 app/controllers/manifests_controller.rb            |   1 +
 app/controllers/media_controller.rb                |   1 -
 app/controllers/public_timelines_controller.rb     |  14 +-
 app/controllers/remote_follow_controller.rb        |  12 +-
 app/controllers/statuses_controller.rb             | 164 ++-------------------
 app/controllers/tags_controller.rb                 |  18 ++-
 app/controllers/well_known/host_meta_controller.rb |   2 +-
 app/controllers/well_known/webfinger_controller.rb |   9 +-
 app/lib/activitypub/activity/announce.rb           |   2 +-
 app/lib/activitypub/activity/create.rb             |   2 +-
 app/lib/activitypub/activity/delete.rb             |   2 +-
 app/lib/activitypub/tag_manager.rb                 |   2 +-
 app/models/status.rb                               |   9 +-
 app/serializers/activitypub/activity_serializer.rb |   3 +
 app/serializers/activitypub/actor_serializer.rb    |   2 +
 .../activitypub/collection_serializer.rb           |   2 +
 app/serializers/activitypub/emoji_serializer.rb    |   2 +
 app/serializers/activitypub/note_serializer.rb     |   2 +
 app/services/process_hashtags_service.rb           |   2 +-
 app/views/statuses/_simple_status.html.haml        |   4 +-
 config/routes.rb                                   |   3 +-
 .../concerns/account_controller_concern_spec.rb    |   2 +-
 spec/controllers/statuses_controller_spec.rb       |   4 +-
 spec/requests/link_headers_spec.rb                 |   8 +-
 41 files changed, 299 insertions(+), 289 deletions(-)
 create mode 100644 app/controllers/activitypub/replies_controller.rb
 create mode 100644 app/controllers/concerns/account_owned_concern.rb
 create mode 100644 app/controllers/concerns/status_controller_concern.rb

(limited to 'app/models')

diff --git a/app/controllers/about_controller.rb b/app/controllers/about_controller.rb
index 52a51fd62..761c7f5cd 100644
--- a/app/controllers/about_controller.rb
+++ b/app/controllers/about_controller.rb
@@ -3,11 +3,11 @@
 class AboutController < ApplicationController
   layout 'public'
 
-  before_action :set_instance_presenter, only: [:show, :more, :terms]
+  before_action :set_body_classes, only: :show
+  before_action :set_instance_presenter
+  before_action :set_expires_in
 
-  def show
-    @hide_navbar = true
-  end
+  def show; end
 
   def more; end
 
@@ -27,4 +27,12 @@ class AboutController < ApplicationController
   def set_instance_presenter
     @instance_presenter = InstancePresenter.new
   end
+
+  def set_body_classes
+    @hide_navbar = true
+  end
+
+  def set_expires_in
+    expires_in 0, public: true
+  end
 end
diff --git a/app/controllers/accounts_controller.rb b/app/controllers/accounts_controller.rb
index 065707378..3184a73cb 100644
--- a/app/controllers/accounts_controller.rb
+++ b/app/controllers/accounts_controller.rb
@@ -6,13 +6,13 @@ class AccountsController < ApplicationController
   include AccountControllerConcern
 
   before_action :set_cache_headers
+  before_action :set_body_classes
 
   def show
     respond_to do |format|
       format.html do
-        mark_cacheable! unless user_signed_in?
+        expires_in 0, public: true unless user_signed_in?
 
-        @body_classes      = 'with-modals'
         @pinned_statuses   = []
         @endorsed_accounts = @account.endorsed_accounts.to_a.sample(4)
 
@@ -32,22 +32,25 @@ class AccountsController < ApplicationController
       end
 
       format.rss do
-        mark_cacheable!
+        expires_in 0, public: true
 
         @statuses = cache_collection(default_statuses.without_reblogs.without_replies.limit(PAGE_SIZE), Status)
         render xml: RSS::AccountSerializer.render(@account, @statuses)
       end
 
       format.json do
-        render_cached_json(['activitypub', 'actor', @account], content_type: 'application/activity+json') do
-          ActiveModelSerializers::SerializableResource.new(@account, serializer: ActivityPub::ActorSerializer, adapter: ActivityPub::Adapter)
-        end
+        expires_in 3.minutes, public: true
+        render json: @account, content_type: 'application/activity+json', serializer: ActivityPub::ActorSerializer, adapter: ActivityPub::Adapter
       end
     end
   end
 
   private
 
+  def set_body_classes
+    @body_classes = 'with-modals'
+  end
+
   def show_pinned_statuses?
     [replies_requested?, media_requested?, tag_requested?, params[:max_id].present?, params[:min_id].present?].none?
   end
diff --git a/app/controllers/activitypub/collections_controller.rb b/app/controllers/activitypub/collections_controller.rb
index 012c3c538..dd2f111b0 100644
--- a/app/controllers/activitypub/collections_controller.rb
+++ b/app/controllers/activitypub/collections_controller.rb
@@ -2,29 +2,19 @@
 
 class ActivityPub::CollectionsController < Api::BaseController
   include SignatureVerification
+  include AccountOwnedConcern
 
-  before_action :set_account
   before_action :set_size
   before_action :set_statuses
   before_action :set_cache_headers
 
   def show
-    render_cached_json(['activitypub', 'collection', @account, params[:id]], content_type: 'application/activity+json') do
-      ActiveModelSerializers::SerializableResource.new(
-        collection_presenter,
-        serializer: ActivityPub::CollectionSerializer,
-        adapter: ActivityPub::Adapter,
-        skip_activities: true
-      )
-    end
+    expires_in 3.minutes, public: true
+    render json: collection_presenter, content_type: 'application/activity+json', serializer: ActivityPub::CollectionSerializer, adapter: ActivityPub::Adapter, skip_activities: true
   end
 
   private
 
-  def set_account
-    @account = Account.find_local!(params[:account_username])
-  end
-
   def set_statuses
     @statuses = scope_for_collection
     @statuses = cache_collection(@statuses, Status)
diff --git a/app/controllers/activitypub/inboxes_controller.rb b/app/controllers/activitypub/inboxes_controller.rb
index e2cd8eaed..9be0676e1 100644
--- a/app/controllers/activitypub/inboxes_controller.rb
+++ b/app/controllers/activitypub/inboxes_controller.rb
@@ -3,8 +3,7 @@
 class ActivityPub::InboxesController < Api::BaseController
   include SignatureVerification
   include JsonLdHelper
-
-  before_action :set_account
+  include AccountOwnedConcern
 
   def create
     if unknown_deleted_account?
@@ -27,8 +26,8 @@ class ActivityPub::InboxesController < Api::BaseController
     false
   end
 
-  def set_account
-    @account = Account.find_local!(params[:account_username]) if params[:account_username]
+  def account_required?
+    params[:account_username].present?
   end
 
   def body
diff --git a/app/controllers/activitypub/outboxes_controller.rb b/app/controllers/activitypub/outboxes_controller.rb
index 5147afbf7..4c0b769f0 100644
--- a/app/controllers/activitypub/outboxes_controller.rb
+++ b/app/controllers/activitypub/outboxes_controller.rb
@@ -4,8 +4,8 @@ class ActivityPub::OutboxesController < Api::BaseController
   LIMIT = 20
 
   include SignatureVerification
+  include AccountOwnedConcern
 
-  before_action :set_account
   before_action :set_statuses
   before_action :set_cache_headers
 
@@ -17,10 +17,6 @@ class ActivityPub::OutboxesController < Api::BaseController
 
   private
 
-  def set_account
-    @account = Account.find_local!(params[:account_username])
-  end
-
   def outbox_presenter
     if page_requested?
       ActivityPub::CollectionPresenter.new(
diff --git a/app/controllers/activitypub/replies_controller.rb b/app/controllers/activitypub/replies_controller.rb
new file mode 100644
index 000000000..99b7b310f
--- /dev/null
+++ b/app/controllers/activitypub/replies_controller.rb
@@ -0,0 +1,68 @@
+# frozen_string_literal: true
+
+class ActivityPub::RepliesController < Api::BaseController
+  include SignatureAuthentication
+  include Authorization
+  include AccountOwnedConcern
+
+  DESCENDANTS_LIMIT = 60
+
+  before_action :set_status
+  before_action :set_cache_headers
+  before_action :set_replies
+
+  def index
+    render json: replies_collection_presenter, serializer: ActivityPub::CollectionSerializer, adapter: ActivityPub::Adapter, content_type: 'application/activity+json', skip_activities: true
+  end
+
+  private
+
+  def set_status
+    @status = @account.statuses.find(params[:status_id])
+    authorize @status, :show?
+  rescue Mastodon::NotPermittedError
+    raise ActiveRecord::RecordNotFound
+  end
+
+  def set_replies
+    @replies = page_params[:other_accounts] ? Status.where.not(account_id: @account.id) : @account.statuses
+    @replies = @replies.where(in_reply_to_id: @status.id, visibility: [:public, :unlisted])
+    @replies = @replies.paginate_by_min_id(DESCENDANTS_LIMIT, params[:min_id])
+  end
+
+  def replies_collection_presenter
+    page = ActivityPub::CollectionPresenter.new(
+      id: account_status_replies_url(@account, @status, page_params),
+      type: :unordered,
+      part_of: account_status_replies_url(@account, @status),
+      next: next_page,
+      items: @replies.map { |status| status.local ? status : status.id }
+    )
+
+    return page if page_requested?
+
+    ActivityPub::CollectionPresenter.new(
+      id: account_status_replies_url(@account, @status),
+      type: :unordered,
+      first: page
+    )
+  end
+
+  def page_requested?
+    params[:page] == 'true'
+  end
+
+  def next_page
+    account_status_replies_url(
+      @account,
+      @status,
+      page: true,
+      min_id: @replies&.last&.id,
+      other_accounts: !(@replies&.last&.account_id == @account.id && @replies.size == DESCENDANTS_LIMIT)
+    )
+  end
+
+  def page_params
+    params_slice(:other_accounts, :min_id).merge(page: true)
+  end
+end
diff --git a/app/controllers/api/proofs_controller.rb b/app/controllers/api/proofs_controller.rb
index a84ad2014..a98599eee 100644
--- a/app/controllers/api/proofs_controller.rb
+++ b/app/controllers/api/proofs_controller.rb
@@ -1,10 +1,9 @@
 # frozen_string_literal: true
 
 class Api::ProofsController < Api::BaseController
-  before_action :set_account
+  include AccountOwnedConcern
+
   before_action :set_provider
-  before_action :check_account_approval
-  before_action :check_account_suspension
 
   def index
     render json: @account, serializer: @provider.serializer_class
@@ -16,15 +15,7 @@ class Api::ProofsController < Api::BaseController
     @provider = ProofProvider.find(params[:provider]) || raise(ActiveRecord::RecordNotFound)
   end
 
-  def set_account
-    @account = Account.find_local!(params[:username])
-  end
-
-  def check_account_approval
-    not_found if @account.user_pending?
-  end
-
-  def check_account_suspension
-    gone if @account.suspended?
+  def username_param
+    params[:username]
   end
 end
diff --git a/app/controllers/application_controller.rb b/app/controllers/application_controller.rb
index bd8000db0..cc8b8e4da 100644
--- a/app/controllers/application_controller.rb
+++ b/app/controllers/application_controller.rb
@@ -154,8 +154,4 @@ class ApplicationController < ActionController::Base
   def set_cache_headers
     response.headers['Vary'] = 'Accept'
   end
-
-  def mark_cacheable!
-    expires_in 0, public: true
-  end
 end
diff --git a/app/controllers/concerns/account_controller_concern.rb b/app/controllers/concerns/account_controller_concern.rb
index 1c422096c..287a930da 100644
--- a/app/controllers/concerns/account_controller_concern.rb
+++ b/app/controllers/concerns/account_controller_concern.rb
@@ -3,24 +3,19 @@
 module AccountControllerConcern
   extend ActiveSupport::Concern
 
+  include AccountOwnedConcern
+
   FOLLOW_PER_PAGE = 12
 
   included do
     layout 'public'
 
-    before_action :set_account
-    before_action :check_account_approval
-    before_action :check_account_suspension
     before_action :set_instance_presenter
     before_action :set_link_headers
   end
 
   private
 
-  def set_account
-    @account = Account.find_local!(username_param)
-  end
-
   def set_instance_presenter
     @instance_presenter = InstancePresenter.new
   end
@@ -29,27 +24,15 @@ module AccountControllerConcern
     response.headers['Link'] = LinkHeader.new(
       [
         webfinger_account_link,
-        atom_account_url_link,
         actor_url_link,
       ]
     )
   end
 
-  def username_param
-    params[:account_username]
-  end
-
   def webfinger_account_link
     [
       webfinger_account_url,
-      [%w(rel lrdd), %w(type application/xrd+xml)],
-    ]
-  end
-
-  def atom_account_url_link
-    [
-      account_url(@account, format: 'atom'),
-      [%w(rel alternate), %w(type application/atom+xml)],
+      [%w(rel lrdd), %w(type application/jrd+json)],
     ]
   end
 
@@ -63,15 +46,4 @@ module AccountControllerConcern
   def webfinger_account_url
     webfinger_url(resource: @account.to_webfinger_s)
   end
-
-  def check_account_approval
-    not_found if @account.user_pending?
-  end
-
-  def check_account_suspension
-    if @account.suspended?
-      expires_in(3.minutes, public: true)
-      gone
-    end
-  end
 end
diff --git a/app/controllers/concerns/account_owned_concern.rb b/app/controllers/concerns/account_owned_concern.rb
new file mode 100644
index 000000000..99c240fe9
--- /dev/null
+++ b/app/controllers/concerns/account_owned_concern.rb
@@ -0,0 +1,33 @@
+# frozen_string_literal: true
+
+module AccountOwnedConcern
+  extend ActiveSupport::Concern
+
+  included do
+    before_action :set_account, if: :account_required?
+    before_action :check_account_approval, if: :account_required?
+    before_action :check_account_suspension, if: :account_required?
+  end
+
+  private
+
+  def account_required?
+    true
+  end
+
+  def set_account
+    @account = Account.find_local!(username_param)
+  end
+
+  def username_param
+    params[:account_username]
+  end
+
+  def check_account_approval
+    not_found if @account.local? && @account.user_pending?
+  end
+
+  def check_account_suspension
+    expires_in(3.minutes, public: true) && gone if @account.suspended?
+  end
+end
diff --git a/app/controllers/concerns/status_controller_concern.rb b/app/controllers/concerns/status_controller_concern.rb
new file mode 100644
index 000000000..62a7cf508
--- /dev/null
+++ b/app/controllers/concerns/status_controller_concern.rb
@@ -0,0 +1,87 @@
+# frozen_string_literal: true
+
+module StatusControllerConcern
+  extend ActiveSupport::Concern
+
+  ANCESTORS_LIMIT         = 40
+  DESCENDANTS_LIMIT       = 60
+  DESCENDANTS_DEPTH_LIMIT = 20
+
+  def create_descendant_thread(starting_depth, statuses)
+    depth = starting_depth + statuses.size
+
+    if depth < DESCENDANTS_DEPTH_LIMIT
+      {
+        statuses: statuses,
+        starting_depth: starting_depth,
+      }
+    else
+      next_status = statuses.pop
+
+      {
+        statuses: statuses,
+        starting_depth: starting_depth,
+        next_status: next_status,
+      }
+    end
+  end
+
+  def set_ancestors
+    @ancestors     = @status.reply? ? cache_collection(@status.ancestors(ANCESTORS_LIMIT, current_account), Status) : []
+    @next_ancestor = @ancestors.size < ANCESTORS_LIMIT ? nil : @ancestors.shift
+  end
+
+  def set_descendants
+    @max_descendant_thread_id   = params[:max_descendant_thread_id]&.to_i
+    @since_descendant_thread_id = params[:since_descendant_thread_id]&.to_i
+
+    descendants = cache_collection(
+      @status.descendants(
+        DESCENDANTS_LIMIT,
+        current_account,
+        @max_descendant_thread_id,
+        @since_descendant_thread_id,
+        DESCENDANTS_DEPTH_LIMIT
+      ),
+      Status
+    )
+
+    @descendant_threads = []
+
+    if descendants.present?
+      statuses       = [descendants.first]
+      starting_depth = 0
+
+      descendants.drop(1).each_with_index do |descendant, index|
+        if descendants[index].id == descendant.in_reply_to_id
+          statuses << descendant
+        else
+          @descendant_threads << create_descendant_thread(starting_depth, statuses)
+
+          # The thread is broken, assume it's a reply to the root status
+          starting_depth = 0
+
+          # ... unless we can find its ancestor in one of the already-processed threads
+          @descendant_threads.reverse_each do |descendant_thread|
+            statuses = descendant_thread[:statuses]
+
+            index = statuses.find_index do |thread_status|
+              thread_status.id == descendant.in_reply_to_id
+            end
+
+            if index.present?
+              starting_depth = descendant_thread[:starting_depth] + index + 1
+              break
+            end
+          end
+
+          statuses = [descendant]
+        end
+      end
+
+      @descendant_threads << create_descendant_thread(starting_depth, statuses)
+    end
+
+    @max_descendant_thread_id = @descendant_threads.pop[:statuses].first.id if descendants.size >= DESCENDANTS_LIMIT
+  end
+end
diff --git a/app/controllers/custom_css_controller.rb b/app/controllers/custom_css_controller.rb
index 6e80feaf8..7f4dcfcfe 100644
--- a/app/controllers/custom_css_controller.rb
+++ b/app/controllers/custom_css_controller.rb
@@ -6,6 +6,7 @@ class CustomCssController < ApplicationController
   before_action :set_cache_headers
 
   def show
+    expires 3.minutes, public: true
     render plain: Setting.custom_css || '', content_type: 'text/css'
   end
 end
diff --git a/app/controllers/emojis_controller.rb b/app/controllers/emojis_controller.rb
index 3feb08132..fe4c19cad 100644
--- a/app/controllers/emojis_controller.rb
+++ b/app/controllers/emojis_controller.rb
@@ -7,9 +7,8 @@ class EmojisController < ApplicationController
   def show
     respond_to do |format|
       format.json do
-        render_cached_json(['activitypub', 'emoji', @emoji], content_type: 'application/activity+json') do
-          ActiveModelSerializers::SerializableResource.new(@emoji, serializer: ActivityPub::EmojiSerializer, adapter: ActivityPub::Adapter)
-        end
+        expires_in 3.minutes, public: true
+        render json: @emoji, content_type: 'application/activity+json', serializer: ActivityPub::EmojiSerializer, adapter: ActivityPub::Adapter
       end
     end
   end
diff --git a/app/controllers/follower_accounts_controller.rb b/app/controllers/follower_accounts_controller.rb
index 415abe10c..8baa64490 100644
--- a/app/controllers/follower_accounts_controller.rb
+++ b/app/controllers/follower_accounts_controller.rb
@@ -8,7 +8,7 @@ class FollowerAccountsController < ApplicationController
   def index
     respond_to do |format|
       format.html do
-        mark_cacheable! unless user_signed_in?
+        expires_in 0, public: true unless user_signed_in?
 
         next if @account.user_hides_network?
 
diff --git a/app/controllers/following_accounts_controller.rb b/app/controllers/following_accounts_controller.rb
index 948725664..4d1ea4594 100644
--- a/app/controllers/following_accounts_controller.rb
+++ b/app/controllers/following_accounts_controller.rb
@@ -8,7 +8,7 @@ class FollowingAccountsController < ApplicationController
   def index
     respond_to do |format|
       format.html do
-        mark_cacheable! unless user_signed_in?
+        expires_in 0, public: true unless user_signed_in?
 
         next if @account.user_hides_network?
 
diff --git a/app/controllers/home_controller.rb b/app/controllers/home_controller.rb
index 85622a7b5..d1c525134 100644
--- a/app/controllers/home_controller.rb
+++ b/app/controllers/home_controller.rb
@@ -21,7 +21,7 @@ class HomeController < ApplicationController
       when 'statuses'
         status = Status.find_by(id: matches[2])
 
-        if status && (status.public_visibility? || status.unlisted_visibility?)
+        if status&.distributable?
           redirect_to(ActivityPub::TagManager.instance.url_for(status))
           return
         end
diff --git a/app/controllers/intents_controller.rb b/app/controllers/intents_controller.rb
index 9f41cf48a..ca89fc7fe 100644
--- a/app/controllers/intents_controller.rb
+++ b/app/controllers/intents_controller.rb
@@ -2,6 +2,7 @@
 
 class IntentsController < ApplicationController
   before_action :check_uri
+
   rescue_from Addressable::URI::InvalidURIError, with: :handle_invalid_uri
 
   def show
diff --git a/app/controllers/manifests_controller.rb b/app/controllers/manifests_controller.rb
index 332d845d8..1e5db4393 100644
--- a/app/controllers/manifests_controller.rb
+++ b/app/controllers/manifests_controller.rb
@@ -4,6 +4,7 @@ class ManifestsController < ApplicationController
   skip_before_action :store_current_location
 
   def show
+    expires_in 3.minutes, public: true
     render json: InstancePresenter.new, serializer: ManifestSerializer
   end
 end
diff --git a/app/controllers/media_controller.rb b/app/controllers/media_controller.rb
index d44b52d26..b3b7519a1 100644
--- a/app/controllers/media_controller.rb
+++ b/app/controllers/media_controller.rb
@@ -31,7 +31,6 @@ class MediaController < ApplicationController
   def verify_permitted_status!
     authorize @media_attachment.status, :show?
   rescue Mastodon::NotPermittedError
-    # Reraise in order to get a 404 instead of a 403 error code
     raise ActiveRecord::RecordNotFound
   end
 
diff --git a/app/controllers/public_timelines_controller.rb b/app/controllers/public_timelines_controller.rb
index 53d4472d8..23506b990 100644
--- a/app/controllers/public_timelines_controller.rb
+++ b/app/controllers/public_timelines_controller.rb
@@ -8,20 +8,16 @@ class PublicTimelinesController < ApplicationController
   before_action :set_instance_presenter
 
   def show
-    respond_to do |format|
-      format.html do
-        @initial_state_json = ActiveModelSerializers::SerializableResource.new(
-          InitialStatePresenter.new(settings: { known_fediverse: Setting.show_known_fediverse_at_about_page }, token: current_session&.token),
-          serializer: InitialStateSerializer
-        ).to_json
-      end
-    end
+    @initial_state_json = ActiveModelSerializers::SerializableResource.new(
+      InitialStatePresenter.new(settings: { known_fediverse: Setting.show_known_fediverse_at_about_page }, token: current_session&.token),
+      serializer: InitialStateSerializer
+    ).to_json
   end
 
   private
 
   def check_enabled
-    raise ActiveRecord::RecordNotFound unless Setting.timeline_preview
+    not_found unless Setting.timeline_preview
   end
 
   def set_body_classes
diff --git a/app/controllers/remote_follow_controller.rb b/app/controllers/remote_follow_controller.rb
index 8ba331cd1..0fb71d335 100644
--- a/app/controllers/remote_follow_controller.rb
+++ b/app/controllers/remote_follow_controller.rb
@@ -1,10 +1,10 @@
 # frozen_string_literal: true
 
 class RemoteFollowController < ApplicationController
+  include AccountOwnedConcern
+
   layout 'modal'
 
-  before_action :set_account
-  before_action :gone, if: :suspended_account?
   before_action :set_body_classes
 
   def new
@@ -32,14 +32,6 @@ class RemoteFollowController < ApplicationController
     { acct: session[:remote_follow] }
   end
 
-  def set_account
-    @account = Account.find_local!(params[:account_username])
-  end
-
-  def suspended_account?
-    @account.suspended?
-  end
-
   def set_body_classes
     @body_classes = 'modal-layout'
     @hide_header  = true
diff --git a/app/controllers/statuses_controller.rb b/app/controllers/statuses_controller.rb
index 776099ca8..13ce5c691 100644
--- a/app/controllers/statuses_controller.rb
+++ b/app/controllers/statuses_controller.rb
@@ -1,24 +1,21 @@
 # frozen_string_literal: true
 
 class StatusesController < ApplicationController
+  include StatusControllerConcern
   include SignatureAuthentication
   include Authorization
-
-  ANCESTORS_LIMIT         = 40
-  DESCENDANTS_LIMIT       = 60
-  DESCENDANTS_DEPTH_LIMIT = 20
+  include AccountOwnedConcern
 
   layout 'public'
 
-  before_action :set_account
   before_action :set_status
   before_action :set_instance_presenter
   before_action :set_link_headers
-  before_action :check_account_suspension
   before_action :redirect_to_original, only: [:show]
   before_action :set_referrer_policy_header, only: [:show]
   before_action :set_cache_headers
-  before_action :set_replies, only: [:replies]
+  before_action :set_body_classes
+  before_action :set_autoplay, only: :embed
 
   content_security_policy only: :embed do |p|
     p.frame_ancestors(false)
@@ -28,25 +25,20 @@ class StatusesController < ApplicationController
     respond_to do |format|
       format.html do
         expires_in 10.seconds, public: true if current_account.nil?
-
-        @body_classes = 'with-modals'
-
         set_ancestors
         set_descendants
       end
 
       format.json do
-        render_cached_json(['activitypub', 'note', @status], content_type: 'application/activity+json', public: @status.distributable?) do
-          ActiveModelSerializers::SerializableResource.new(@status, serializer: ActivityPub::NoteSerializer, adapter: ActivityPub::Adapter)
-        end
+        expires_in 3.minutes, public: @status.distributable?
+        render json: @status, content_type: 'application/activity+json', serializer: ActivityPub::NoteSerializer, adapter: ActivityPub::Adapter
       end
     end
   end
 
   def activity
-    render_cached_json(['activitypub', 'activity', @status], content_type: 'application/activity+json', public: @status.distributable?) do
-      ActiveModelSerializers::SerializableResource.new(@status, serializer: ActivityPub::ActivitySerializer, adapter: ActivityPub::Adapter)
-    end
+    expires_in 3.minutes, public: @status.distributable?
+    render json: @status, content_type: 'application/activity+json', serializer: ActivityPub::ActivitySerializer, adapter: ActivityPub::Adapter
   end
 
   def embed
@@ -54,120 +46,14 @@ class StatusesController < ApplicationController
 
     expires_in 180, public: true
     response.headers['X-Frame-Options'] = 'ALLOWALL'
-    @autoplay = ActiveModel::Type::Boolean.new.cast(params[:autoplay])
 
     render layout: 'embedded'
   end
 
-  def replies
-    render json: replies_collection_presenter,
-           serializer: ActivityPub::CollectionSerializer,
-           adapter: ActivityPub::Adapter,
-           content_type: 'application/activity+json',
-           skip_activities: true
-  end
-
   private
 
-  def replies_collection_presenter
-    page = ActivityPub::CollectionPresenter.new(
-      id: replies_account_status_url(@account, @status, page_params),
-      type: :unordered,
-      part_of: replies_account_status_url(@account, @status),
-      next: next_page,
-      items: @replies.map { |status| status.local ? status : status.id }
-    )
-    if page_requested?
-      page
-    else
-      ActivityPub::CollectionPresenter.new(
-        id: replies_account_status_url(@account, @status),
-        type: :unordered,
-        first: page
-      )
-    end
-  end
-
-  def create_descendant_thread(starting_depth, statuses)
-    depth = starting_depth + statuses.size
-
-    if depth < DESCENDANTS_DEPTH_LIMIT
-      {
-        statuses: statuses,
-        starting_depth: starting_depth,
-      }
-    else
-      next_status = statuses.pop
-
-      {
-        statuses: statuses,
-        starting_depth: starting_depth,
-        next_status: next_status,
-      }
-    end
-  end
-
-  def set_account
-    @account = Account.find_local!(params[:account_username])
-  end
-
-  def set_ancestors
-    @ancestors     = @status.reply? ? cache_collection(@status.ancestors(ANCESTORS_LIMIT, current_account), Status) : []
-    @next_ancestor = @ancestors.size < ANCESTORS_LIMIT ? nil : @ancestors.shift
-  end
-
-  def set_descendants
-    @max_descendant_thread_id   = params[:max_descendant_thread_id]&.to_i
-    @since_descendant_thread_id = params[:since_descendant_thread_id]&.to_i
-
-    descendants = cache_collection(
-      @status.descendants(
-        DESCENDANTS_LIMIT,
-        current_account,
-        @max_descendant_thread_id,
-        @since_descendant_thread_id,
-        DESCENDANTS_DEPTH_LIMIT
-      ),
-      Status
-    )
-
-    @descendant_threads = []
-
-    if descendants.present?
-      statuses       = [descendants.first]
-      starting_depth = 0
-
-      descendants.drop(1).each_with_index do |descendant, index|
-        if descendants[index].id == descendant.in_reply_to_id
-          statuses << descendant
-        else
-          @descendant_threads << create_descendant_thread(starting_depth, statuses)
-
-          # The thread is broken, assume it's a reply to the root status
-          starting_depth = 0
-
-          # ... unless we can find its ancestor in one of the already-processed threads
-          @descendant_threads.reverse_each do |descendant_thread|
-            statuses = descendant_thread[:statuses]
-
-            index = statuses.find_index do |thread_status|
-              thread_status.id == descendant.in_reply_to_id
-            end
-
-            if index.present?
-              starting_depth = descendant_thread[:starting_depth] + index + 1
-              break
-            end
-          end
-
-          statuses = [descendant]
-        end
-      end
-
-      @descendant_threads << create_descendant_thread(starting_depth, statuses)
-    end
-
-    @max_descendant_thread_id = @descendant_threads.pop[:statuses].first.id if descendants.size >= DESCENDANTS_LIMIT
+  def set_body_classes
+    @body_classes = 'with-modals'
   end
 
   def set_link_headers
@@ -185,39 +71,15 @@ class StatusesController < ApplicationController
     @instance_presenter = InstancePresenter.new
   end
 
-  def check_account_suspension
-    gone if @account.suspended?
-  end
-
   def redirect_to_original
     redirect_to ActivityPub::TagManager.instance.url_for(@status.reblog) if @status.reblog?
   end
 
   def set_referrer_policy_header
-    return if @status.public_visibility? || @status.unlisted_visibility?
-    response.headers['Referrer-Policy'] = 'origin'
-  end
-
-  def page_requested?
-    params[:page] == 'true'
-  end
-
-  def set_replies
-    @replies = page_params[:other_accounts] ? Status.where.not(account_id: @account.id) : @account.statuses
-    @replies = @replies.where(in_reply_to_id: @status.id, visibility: [:public, :unlisted])
-    @replies = @replies.paginate_by_min_id(DESCENDANTS_LIMIT, params[:min_id])
-  end
-
-  def next_page
-    last_reply = @replies.last
-    return if last_reply.nil?
-    same_account = last_reply.account_id == @account.id
-    return unless same_account || @replies.size == DESCENDANTS_LIMIT
-    same_account = false unless @replies.size == DESCENDANTS_LIMIT
-    replies_account_status_url(@account, @status, page: true, min_id: last_reply.id, other_accounts: !same_account)
+    response.headers['Referrer-Policy'] = 'origin' unless @status.distributable?
   end
 
-  def page_params
-    { page: true, other_accounts: params[:other_accounts], min_id: params[:min_id] }.compact
+  def set_autoplay
+    @autoplay = truthy_param?(:autoplay)
   end
 end
diff --git a/app/controllers/tags_controller.rb b/app/controllers/tags_controller.rb
index 66b184901..2ecce0ca2 100644
--- a/app/controllers/tags_controller.rb
+++ b/app/controllers/tags_controller.rb
@@ -5,14 +5,15 @@ class TagsController < ApplicationController
 
   layout 'public'
 
+  before_action :set_tag
   before_action :set_body_classes
   before_action :set_instance_presenter
 
   def show
-    @tag = Tag.find_normalized!(params[:id])
-
     respond_to do |format|
       format.html do
+        expires_in 0, public: true
+
         @initial_state_json = ActiveModelSerializers::SerializableResource.new(
           InitialStatePresenter.new(settings: {}, token: current_session&.token),
           serializer: InitialStateSerializer
@@ -20,6 +21,8 @@ class TagsController < ApplicationController
       end
 
       format.rss do
+        expires_in 0, public: true
+
         @statuses = HashtagQueryService.new.call(@tag, params.slice(:any, :all, :none)).limit(PAGE_SIZE)
         @statuses = cache_collection(@statuses, Status)
 
@@ -27,19 +30,22 @@ class TagsController < ApplicationController
       end
 
       format.json do
+        expires_in 3.minutes, public: true
+
         @statuses = HashtagQueryService.new.call(@tag, params.slice(:any, :all, :none), current_account, params[:local]).paginate_by_max_id(PAGE_SIZE, params[:max_id])
         @statuses = cache_collection(@statuses, Status)
 
-        render json: collection_presenter,
-               serializer: ActivityPub::CollectionSerializer,
-               adapter: ActivityPub::Adapter,
-               content_type: 'application/activity+json'
+        render json: collection_presenter, serializer: ActivityPub::CollectionSerializer, adapter: ActivityPub::Adapter, content_type: 'application/activity+json'
       end
     end
   end
 
   private
 
+  def set_tag
+    @tag = Tag.find_normalized!(params[:id])
+  end
+
   def set_body_classes
     @body_classes = 'with-modals'
   end
diff --git a/app/controllers/well_known/host_meta_controller.rb b/app/controllers/well_known/host_meta_controller.rb
index 5fb70288a..2e9298c4a 100644
--- a/app/controllers/well_known/host_meta_controller.rb
+++ b/app/controllers/well_known/host_meta_controller.rb
@@ -13,7 +13,7 @@ module WellKnown
         format.xml { render content_type: 'application/xrd+xml' }
       end
 
-      expires_in(3.days, public: true)
+      expires_in 3.days, public: true
     end
   end
 end
diff --git a/app/controllers/well_known/webfinger_controller.rb b/app/controllers/well_known/webfinger_controller.rb
index 28654b61d..53f7f1e27 100644
--- a/app/controllers/well_known/webfinger_controller.rb
+++ b/app/controllers/well_known/webfinger_controller.rb
@@ -19,7 +19,7 @@ module WellKnown
         end
       end
 
-      expires_in(3.days, public: true)
+      expires_in 3.days, public: true
     rescue ActiveRecord::RecordNotFound
       head 404
     end
@@ -27,12 +27,9 @@ module WellKnown
     private
 
     def username_from_resource
-      resource_user = resource_param
-
+      resource_user    = resource_param
       username, domain = resource_user.split('@')
-      if Rails.configuration.x.alternate_domains.include?(domain)
-        resource_user = "#{username}@#{Rails.configuration.x.local_domain}"
-      end
+      resource_user    = "#{username}@#{Rails.configuration.x.local_domain}" if Rails.configuration.x.alternate_domains.include?(domain)
 
       WebfingerResource.new(resource_user).username
     end
diff --git a/app/lib/activitypub/activity/announce.rb b/app/lib/activitypub/activity/announce.rb
index 1aa6ee9ec..34c646668 100644
--- a/app/lib/activitypub/activity/announce.rb
+++ b/app/lib/activitypub/activity/announce.rb
@@ -40,7 +40,7 @@ class ActivityPub::Activity::Announce < ActivityPub::Activity
   end
 
   def announceable?(status)
-    status.account_id == @account.id || status.public_visibility? || status.unlisted_visibility?
+    status.account_id == @account.id || status.distributable?
   end
 
   def related_to_local_activity?
diff --git a/app/lib/activitypub/activity/create.rb b/app/lib/activitypub/activity/create.rb
index 00f0dd42d..5849c20d7 100644
--- a/app/lib/activitypub/activity/create.rb
+++ b/app/lib/activitypub/activity/create.rb
@@ -42,7 +42,7 @@ class ActivityPub::Activity::Create < ActivityPub::Activity
     resolve_thread(@status)
     fetch_replies(@status)
     distribute(@status)
-    forward_for_reply if @status.public_visibility? || @status.unlisted_visibility?
+    forward_for_reply if @status.distributable?
   end
 
   def find_existing_status
diff --git a/app/lib/activitypub/activity/delete.rb b/app/lib/activitypub/activity/delete.rb
index 0eb14b89c..1f2b40c15 100644
--- a/app/lib/activitypub/activity/delete.rb
+++ b/app/lib/activitypub/activity/delete.rb
@@ -31,7 +31,7 @@ class ActivityPub::Activity::Delete < ActivityPub::Activity
 
     return if @status.nil?
 
-    if @status.public_visibility? || @status.unlisted_visibility?
+    if @status.distributable?
       forward_for_reply
       forward_for_reblogs
     end
diff --git a/app/lib/activitypub/tag_manager.rb b/app/lib/activitypub/tag_manager.rb
index 595291342..4d452f290 100644
--- a/app/lib/activitypub/tag_manager.rb
+++ b/app/lib/activitypub/tag_manager.rb
@@ -51,7 +51,7 @@ class ActivityPub::TagManager
   def replies_uri_for(target, page_params = nil)
     raise ArgumentError, 'target must be a local activity' unless %i(note comment activity).include?(target.object_type) && target.local?
 
-    replies_account_status_url(target.account, target, page_params)
+    account_status_replies_url(target.account, target, page_params)
   end
 
   # Primary audience of a status
diff --git a/app/models/status.rb b/app/models/status.rb
index 906756e85..6f1e35e4a 100644
--- a/app/models/status.rb
+++ b/app/models/status.rb
@@ -193,7 +193,7 @@ class Status < ApplicationRecord
   end
 
   def hidden?
-    private_visibility? || direct_visibility? || limited_visibility?
+    !distributable?
   end
 
   def distributable?
@@ -446,7 +446,8 @@ class Status < ApplicationRecord
   end
 
   def update_statistics
-    return unless public_visibility? || unlisted_visibility?
+    return unless distributable?
+
     ActivityTracker.increment('activity:statuses:local')
   end
 
@@ -455,7 +456,7 @@ class Status < ApplicationRecord
 
     account&.increment_count!(:statuses_count)
     reblog&.increment_count!(:reblogs_count) if reblog?
-    thread&.increment_count!(:replies_count) if in_reply_to_id.present? && (public_visibility? || unlisted_visibility?)
+    thread&.increment_count!(:replies_count) if in_reply_to_id.present? && distributable?
   end
 
   def decrement_counter_caches
@@ -463,7 +464,7 @@ class Status < ApplicationRecord
 
     account&.decrement_count!(:statuses_count)
     reblog&.decrement_count!(:reblogs_count) if reblog?
-    thread&.decrement_count!(:replies_count) if in_reply_to_id.present? && (public_visibility? || unlisted_visibility?)
+    thread&.decrement_count!(:replies_count) if in_reply_to_id.present? && distributable?
   end
 
   def unlink_from_conversations
diff --git a/app/serializers/activitypub/activity_serializer.rb b/app/serializers/activitypub/activity_serializer.rb
index c06d5c87c..fdedbc9d1 100644
--- a/app/serializers/activitypub/activity_serializer.rb
+++ b/app/serializers/activitypub/activity_serializer.rb
@@ -1,9 +1,12 @@
 # frozen_string_literal: true
 
 class ActivityPub::ActivitySerializer < ActivityPub::Serializer
+  cache key: 'activity', expires_in: 3.minutes
+
   attributes :id, :type, :actor, :published, :to, :cc
 
   has_one :proper, key: :object, serializer: ActivityPub::NoteSerializer, if: :serialize_object?
+
   attribute :proper_uri, key: :object, unless: :serialize_object?
   attribute :atom_uri, if: :announce?
 
diff --git a/app/serializers/activitypub/actor_serializer.rb b/app/serializers/activitypub/actor_serializer.rb
index 0644219fb..ab7be27f6 100644
--- a/app/serializers/activitypub/actor_serializer.rb
+++ b/app/serializers/activitypub/actor_serializer.rb
@@ -3,6 +3,8 @@
 class ActivityPub::ActorSerializer < ActivityPub::Serializer
   include RoutingHelper
 
+  cache key: 'actor', expires_in: 3.minutes
+
   context :security
 
   context_extensions :manually_approves_followers, :featured, :also_known_as,
diff --git a/app/serializers/activitypub/collection_serializer.rb b/app/serializers/activitypub/collection_serializer.rb
index da1ba735f..9dd8134d3 100644
--- a/app/serializers/activitypub/collection_serializer.rb
+++ b/app/serializers/activitypub/collection_serializer.rb
@@ -7,6 +7,8 @@ class ActivityPub::CollectionSerializer < ActivityPub::Serializer
     super
   end
 
+  cache key: 'collection', expires_in: 3.minutes
+
   attribute :id, if: -> { object.id.present? }
   attribute :type
   attribute :total_items, if: -> { object.size.present? }
diff --git a/app/serializers/activitypub/emoji_serializer.rb b/app/serializers/activitypub/emoji_serializer.rb
index 4dc38f3ea..08df25d7d 100644
--- a/app/serializers/activitypub/emoji_serializer.rb
+++ b/app/serializers/activitypub/emoji_serializer.rb
@@ -3,6 +3,8 @@
 class ActivityPub::EmojiSerializer < ActivityPub::Serializer
   include RoutingHelper
 
+  cache key: 'emoji', expires_in: 3.minutes
+
   context_extensions :emoji
 
   attributes :id, :type, :name, :updated
diff --git a/app/serializers/activitypub/note_serializer.rb b/app/serializers/activitypub/note_serializer.rb
index 67f596e78..87acc5429 100644
--- a/app/serializers/activitypub/note_serializer.rb
+++ b/app/serializers/activitypub/note_serializer.rb
@@ -1,6 +1,8 @@
 # frozen_string_literal: true
 
 class ActivityPub::NoteSerializer < ActivityPub::Serializer
+  cache key: 'note', expires_in: 3.minutes
+
   context_extensions :atom_uri, :conversation, :sensitive,
                      :hashtag, :emoji, :focal_point, :blurhash
 
diff --git a/app/services/process_hashtags_service.rb b/app/services/process_hashtags_service.rb
index d5ec076a8..b6974e598 100644
--- a/app/services/process_hashtags_service.rb
+++ b/app/services/process_hashtags_service.rb
@@ -14,7 +14,7 @@ class ProcessHashtagsService < BaseService
       TrendingTags.record_use!(tag, status.account, status.created_at) if status.public_visibility?
     end
 
-    return unless status.public_visibility? || status.unlisted_visibility?
+    return unless status.distributable?
 
     status.account.featured_tags.where(tag_id: records.map(&:id)).each do |featured_tag|
       featured_tag.increment(status.created_at)
diff --git a/app/views/statuses/_simple_status.html.haml b/app/views/statuses/_simple_status.html.haml
index 11220dfcb..38fde1be8 100644
--- a/app/views/statuses/_simple_status.html.haml
+++ b/app/views/statuses/_simple_status.html.haml
@@ -50,9 +50,9 @@
           = fa_icon 'reply-all fw'
       .status__action-bar__counter__label= obscured_counter status.replies_count
     = link_to remote_interaction_path(status, type: :reblog), class: 'status__action-bar-button icon-button modal-button', style: 'font-size: 18px; width: 23.1429px; height: 23.1429px; line-height: 23.15px;' do
-      - if status.public_visibility? || status.unlisted_visibility?
+      - if status.distributable?
         = fa_icon 'retweet fw'
-      - elsif status.private_visibility?
+      - elsif status.private_visibility? || status.limited_visibility?
         = fa_icon 'lock fw'
       - else
         = fa_icon 'envelope fw'
diff --git a/config/routes.rb b/config/routes.rb
index 69b495a96..115e7bb44 100644
--- a/config/routes.rb
+++ b/config/routes.rb
@@ -52,8 +52,9 @@ Rails.application.routes.draw do
       member do
         get :activity
         get :embed
-        get :replies
       end
+
+      resources :replies, only: [:index], module: :activitypub
     end
 
     resources :followers, only: [:index], controller: :follower_accounts
diff --git a/spec/controllers/concerns/account_controller_concern_spec.rb b/spec/controllers/concerns/account_controller_concern_spec.rb
index ea2b4a2a1..7ea214a7d 100644
--- a/spec/controllers/concerns/account_controller_concern_spec.rb
+++ b/spec/controllers/concerns/account_controller_concern_spec.rb
@@ -41,7 +41,7 @@ describe ApplicationController, type: :controller do
     it 'sets link headers' do
       account = Fabricate(:account, username: 'username', user: Fabricate(:user))
       get 'success', params: { account_username: 'username' }
-      expect(response.headers['Link'].to_s).to eq '<http://test.host/.well-known/webfinger?resource=acct%3Ausername%40cb6e6126.ngrok.io>; rel="lrdd"; type="application/xrd+xml", <http://test.host/users/username.atom>; rel="alternate"; type="application/atom+xml", <https://cb6e6126.ngrok.io/users/username>; rel="alternate"; type="application/activity+json"'
+      expect(response.headers['Link'].to_s).to eq '<http://test.host/.well-known/webfinger?resource=acct%3Ausername%40cb6e6126.ngrok.io>; rel="lrdd"; type="application/jrd+json", <https://cb6e6126.ngrok.io/users/username>; rel="alternate"; type="application/activity+json"'
     end
 
     it 'returns http success' do
diff --git a/spec/controllers/statuses_controller_spec.rb b/spec/controllers/statuses_controller_spec.rb
index 95e5c363c..6905dae10 100644
--- a/spec/controllers/statuses_controller_spec.rb
+++ b/spec/controllers/statuses_controller_spec.rb
@@ -92,7 +92,7 @@ describe StatusesController do
       end
 
       it 'assigns @max_descendant_thread_id for the last thread if it is hitting the status limit' do
-        stub_const 'StatusesController::DESCENDANTS_LIMIT', 1
+        stub_const 'StatusControllerConcern::DESCENDANTS_LIMIT', 1
         status = Fabricate(:status)
         child = Fabricate(:status, in_reply_to_id: status.id)
 
@@ -103,7 +103,7 @@ describe StatusesController do
       end
 
       it 'assigns @descendant_threads for threads with :next_status key if they are hitting the depth limit' do
-        stub_const 'StatusesController::DESCENDANTS_DEPTH_LIMIT', 2
+        stub_const 'StatusControllerConcern::DESCENDANTS_DEPTH_LIMIT', 2
         status = Fabricate(:status)
         child0 = Fabricate(:status, in_reply_to_id: status.id)
         child1 = Fabricate(:status, in_reply_to_id: child0.id)
diff --git a/spec/requests/link_headers_spec.rb b/spec/requests/link_headers_spec.rb
index 3dc408d92..712ee262b 100644
--- a/spec/requests/link_headers_spec.rb
+++ b/spec/requests/link_headers_spec.rb
@@ -11,16 +11,16 @@ describe 'Link headers' do
     end
 
     it 'contains webfinger url in link header' do
-      link_header = link_header_with_type('application/xrd+xml')
+      link_header = link_header_with_type('application/jrd+json')
 
       expect(link_header.href).to match 'http://www.example.com/.well-known/webfinger?resource=acct%3Atest%40cb6e6126.ngrok.io'
       expect(link_header.attr_pairs.first).to eq %w(rel lrdd)
     end
 
-    it 'contains atom url in link header' do
-      link_header = link_header_with_type('application/atom+xml')
+    it 'contains activitypub url in link header' do
+      link_header = link_header_with_type('application/activity+json')
 
-      expect(link_header.href).to eq 'http://www.example.com/users/test.atom'
+      expect(link_header.href).to eq 'https://cb6e6126.ngrok.io/users/test'
       expect(link_header.attr_pairs.first).to eq %w(rel alternate)
     end
 
-- 
cgit 


From 1e7187f2a8e0b9ffe4e7d6b06e9f70674c50471e Mon Sep 17 00:00:00 2001
From: ThibG <thib@sitedethib.com>
Date: Mon, 8 Jul 2019 18:17:22 +0200
Subject: Fix Status.remote scope matching *all* statuses (#11265)

---
 app/models/status.rb | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

(limited to 'app/models')

diff --git a/app/models/status.rb b/app/models/status.rb
index 6f1e35e4a..23682c84b 100644
--- a/app/models/status.rb
+++ b/app/models/status.rb
@@ -75,7 +75,7 @@ class Status < ApplicationRecord
   default_scope { recent }
 
   scope :recent, -> { reorder(id: :desc) }
-  scope :remote, -> { where(local: false).or(where.not(uri: nil)) }
+  scope :remote, -> { where(local: false).where.not(uri: nil) }
   scope :local,  -> { where(local: true).or(where(uri: nil)) }
 
   scope :without_replies, -> { where('statuses.reply = FALSE OR statuses.in_reply_to_account_id = statuses.account_id') }
-- 
cgit 


From 6ff67be0f6e79ec403e08c69717ee8c89451c70e Mon Sep 17 00:00:00 2001
From: Eugen Rochko <eugen@zeonfederated.com>
Date: Sat, 13 Jul 2019 16:45:50 +0200
Subject: Add a spam check (#11217)

* Add a spam check

* Use Nilsimsa to generate locality-sensitive hashes and compare using Levenshtein distance

* Add more tests

* Add exemption when the message is a reply to something that mentions the sender

* Use Nilsimsa Compare Value instead of Levenshtein distance

* Use MD5 for messages shorter than 10 characters

* Add message to automated report, do not add non-public statuses to
automated report, add trust level to accounts and make unsilencing
raise the trust level to prevent repeated spam checks on that account

* Expire spam check data after 3 months

* Add support for local statuses, reduce expiration to 1 week, always create a report

* Add content warnings to the spam check and exempt empty statuses

* Change Nilsimsa threshold to 95 and make sure removed statuses are removed from the spam check

* Add all matched statuses into automatic report
---
 Gemfile                                            |   1 +
 Gemfile.lock                                       |   8 +
 app/lib/activitypub/activity/create.rb             |  13 ++
 app/lib/spam_check.rb                              | 169 +++++++++++++++++++++
 app/models/account.rb                              |  18 ++-
 app/services/remove_status_service.rb              |   5 +
 config/locales/en.yml                              |   2 +
 .../20190701022101_add_trust_level_to_accounts.rb  |   5 +
 db/schema.rb                                       |   1 +
 spec/lib/spam_check_spec.rb                        | 160 +++++++++++++++++++
 10 files changed, 377 insertions(+), 5 deletions(-)
 create mode 100644 app/lib/spam_check.rb
 create mode 100644 db/migrate/20190701022101_add_trust_level_to_accounts.rb
 create mode 100644 spec/lib/spam_check_spec.rb

(limited to 'app/models')

diff --git a/Gemfile b/Gemfile
index 613515628..15334678b 100644
--- a/Gemfile
+++ b/Gemfile
@@ -58,6 +58,7 @@ gem 'idn-ruby', require: 'idn'
 gem 'kaminari', '~> 1.1'
 gem 'link_header', '~> 0.0'
 gem 'mime-types', '~> 3.2', require: 'mime/types/columnar'
+gem 'nilsimsa', git: 'https://github.com/witgo/nilsimsa', ref: 'fd184883048b922b176939f851338d0a4971a532'
 gem 'nokogiri', '~> 1.10'
 gem 'nsa', '~> 0.2'
 gem 'oj', '~> 3.7'
diff --git a/Gemfile.lock b/Gemfile.lock
index 340bbcdd8..c3198b7d9 100644
--- a/Gemfile.lock
+++ b/Gemfile.lock
@@ -12,6 +12,13 @@ GIT
   specs:
     http_parser.rb (0.6.1)
 
+GIT
+  remote: https://github.com/witgo/nilsimsa
+  revision: fd184883048b922b176939f851338d0a4971a532
+  ref: fd184883048b922b176939f851338d0a4971a532
+  specs:
+    nilsimsa (1.1.2)
+
 GEM
   remote: https://rubygems.org/
   specs:
@@ -704,6 +711,7 @@ DEPENDENCIES
   microformats (~> 4.1)
   mime-types (~> 3.2)
   net-ldap (~> 0.10)
+  nilsimsa!
   nokogiri (~> 1.10)
   nsa (~> 0.2)
   oj (~> 3.7)
diff --git a/app/lib/activitypub/activity/create.rb b/app/lib/activitypub/activity/create.rb
index 5849c20d7..56c24680a 100644
--- a/app/lib/activitypub/activity/create.rb
+++ b/app/lib/activitypub/activity/create.rb
@@ -41,6 +41,7 @@ class ActivityPub::Activity::Create < ActivityPub::Activity
 
     resolve_thread(@status)
     fetch_replies(@status)
+    check_for_spam
     distribute(@status)
     forward_for_reply if @status.distributable?
   end
@@ -406,6 +407,18 @@ class ActivityPub::Activity::Create < ActivityPub::Activity
     Account.local.where(username: local_usernames).exists?
   end
 
+  def check_for_spam
+    spam_check = SpamCheck.new(@status)
+
+    return if spam_check.skip?
+
+    if spam_check.spam?
+      spam_check.flag!
+    else
+      spam_check.remember!
+    end
+  end
+
   def forward_for_reply
     return unless @json['signature'].present? && reply_to_local?
     ActivityPub::RawDistributionWorker.perform_async(Oj.dump(@json), replied_to_status.account_id, [@account.preferred_inbox_url])
diff --git a/app/lib/spam_check.rb b/app/lib/spam_check.rb
new file mode 100644
index 000000000..923d48a02
--- /dev/null
+++ b/app/lib/spam_check.rb
@@ -0,0 +1,169 @@
+# frozen_string_literal: true
+
+class SpamCheck
+  include Redisable
+  include ActionView::Helpers::TextHelper
+
+  NILSIMSA_COMPARE_THRESHOLD = 95
+  NILSIMSA_MIN_SIZE          = 10
+  EXPIRE_SET_AFTER           = 1.week.seconds
+
+  def initialize(status)
+    @account = status.account
+    @status  = status
+  end
+
+  def skip?
+    already_flagged? || trusted? || no_unsolicited_mentions? || solicited_reply?
+  end
+
+  def spam?
+    if insufficient_data?
+      false
+    elsif nilsimsa?
+      any_other_digest?('nilsimsa') { |_, other_digest| nilsimsa_compare_value(digest, other_digest) >= NILSIMSA_COMPARE_THRESHOLD }
+    else
+      any_other_digest?('md5') { |_, other_digest| other_digest == digest }
+    end
+  end
+
+  def flag!
+    auto_silence_account!
+    auto_report_status!
+  end
+
+  def remember!
+    # The scores in sorted sets don't actually have enough bits to hold an exact
+    # value of our snowflake IDs, so we use it only for its ordering property. To
+    # get the correct status ID back, we have to save it in the string value
+
+    redis.zadd(redis_key, @status.id, digest_with_algorithm)
+    redis.zremrangebyrank(redis_key, '0', '-10')
+    redis.expire(redis_key, EXPIRE_SET_AFTER)
+  end
+
+  def reset!
+    redis.del(redis_key)
+  end
+
+  def hashable_text
+    return @hashable_text if defined?(@hashable_text)
+
+    @hashable_text = @status.text
+    @hashable_text = remove_mentions(@hashable_text)
+    @hashable_text = strip_tags(@hashable_text) unless @status.local?
+    @hashable_text = normalize_unicode(@status.spoiler_text + ' ' + @hashable_text)
+    @hashable_text = remove_whitespace(@hashable_text)
+  end
+
+  def insufficient_data?
+    hashable_text.blank?
+  end
+
+  def digest
+    @digest ||= begin
+      if nilsimsa?
+        Nilsimsa.new(hashable_text).hexdigest
+      else
+        Digest::MD5.hexdigest(hashable_text)
+      end
+    end
+  end
+
+  def digest_with_algorithm
+    if nilsimsa?
+      ['nilsimsa', digest, @status.id].join(':')
+    else
+      ['md5', digest, @status.id].join(':')
+    end
+  end
+
+  private
+
+  def remove_mentions(text)
+    return text.gsub(Account::MENTION_RE, '') if @status.local?
+
+    Nokogiri::HTML.fragment(text).tap do |html|
+      mentions = @status.mentions.map { |mention| ActivityPub::TagManager.instance.url_for(mention.account) }
+
+      html.traverse do |element|
+        element.unlink if element.name == 'a' && mentions.include?(element['href'])
+      end
+    end.to_s
+  end
+
+  def normalize_unicode(text)
+    text.unicode_normalize(:nfkc).downcase
+  end
+
+  def remove_whitespace(text)
+    text.gsub(/\s+/, ' ').strip
+  end
+
+  def auto_silence_account!
+    @account.silence!
+  end
+
+  def auto_report_status!
+    status_ids = Status.where(visibility: %i(public unlisted)).where(id: matching_status_ids).pluck(:id) + [@status.id] if @status.distributable?
+    ReportService.new.call(Account.representative, @account, status_ids: status_ids, comment: I18n.t('spam_check.spam_detected_and_silenced'))
+  end
+
+  def already_flagged?
+    @account.silenced?
+  end
+
+  def trusted?
+    @account.trust_level > Account::TRUST_LEVELS[:untrusted]
+  end
+
+  def no_unsolicited_mentions?
+    @status.mentions.all? { |mention| mention.silent? || (!@account.local? && !mention.account.local?) || mention.account.following?(@account) }
+  end
+
+  def solicited_reply?
+    !@status.thread.nil? && @status.thread.mentions.where(account: @account).exists?
+  end
+
+  def nilsimsa_compare_value(first, second)
+    first  = [first].pack('H*')
+    second = [second].pack('H*')
+    bits   = 0
+
+    0.upto(31) do |i|
+      bits += Nilsimsa::POPC[255 & (first[i].ord ^ second[i].ord)].ord
+    end
+
+    128 - bits # -128 <= Nilsimsa Compare Value <= 128
+  end
+
+  def nilsimsa?
+    hashable_text.size > NILSIMSA_MIN_SIZE
+  end
+
+  def other_digests
+    redis.zrange(redis_key, 0, -1)
+  end
+
+  def any_other_digest?(filter_algorithm)
+    other_digests.any? do |record|
+      algorithm, other_digest, status_id = record.split(':')
+
+      next unless algorithm == filter_algorithm
+
+      yield algorithm, other_digest, status_id
+    end
+  end
+
+  def matching_status_ids
+    if nilsimsa?
+      other_digests.select { |record| record.start_with?('nilsimsa') && nilsimsa_compare_value(digest, record.split(':')[1]) >= NILSIMSA_COMPARE_THRESHOLD }.map { |record| record.split(':')[2] }.compact
+    else
+      other_digests.select { |record| record.start_with?('md5') && record.split(':')[1] == digest }.map { |record| record.split(':')[2] }.compact
+    end
+  end
+
+  def redis_key
+    @redis_key ||= "spam_check:#{@account.id}"
+  end
+end
diff --git a/app/models/account.rb b/app/models/account.rb
index d6772eb98..a22b7fd7c 100644
--- a/app/models/account.rb
+++ b/app/models/account.rb
@@ -45,6 +45,7 @@
 #  also_known_as           :string           is an Array
 #  silenced_at             :datetime
 #  suspended_at            :datetime
+#  trust_level             :integer
 #
 
 class Account < ApplicationRecord
@@ -62,6 +63,11 @@ class Account < ApplicationRecord
   include AccountCounters
   include DomainNormalizable
 
+  TRUST_LEVELS = {
+    untrusted: 0,
+    trusted: 1,
+  }.freeze
+
   enum protocol: [:ostatus, :activitypub]
 
   validates :username, presence: true
@@ -163,6 +169,10 @@ class Account < ApplicationRecord
     last_webfingered_at.nil? || last_webfingered_at <= 1.day.ago
   end
 
+  def trust_level
+    self[:trust_level] || 0
+  end
+
   def refresh!
     ResolveAccountService.new.call(acct) unless local?
   end
@@ -171,21 +181,19 @@ class Account < ApplicationRecord
     silenced_at.present?
   end
 
-  def silence!(date = nil)
-    date ||= Time.now.utc
+  def silence!(date = Time.now.utc)
     update!(silenced_at: date)
   end
 
   def unsilence!
-    update!(silenced_at: nil)
+    update!(silenced_at: nil, trust_level: trust_level == TRUST_LEVELS[:untrusted] ? TRUST_LEVELS[:trusted] : trust_level)
   end
 
   def suspended?
     suspended_at.present?
   end
 
-  def suspend!(date = nil)
-    date ||= Time.now.utc
+  def suspend!(date = Time.now.utc)
     transaction do
       user&.disable! if local?
       update!(suspended_at: date)
diff --git a/app/services/remove_status_service.rb b/app/services/remove_status_service.rb
index 6311971ff..a69fce8b8 100644
--- a/app/services/remove_status_service.rb
+++ b/app/services/remove_status_service.rb
@@ -23,6 +23,7 @@ class RemoveStatusService < BaseService
         remove_from_hashtags
         remove_from_public
         remove_from_media if status.media_attachments.any?
+        remove_from_spam_check
 
         @status.destroy!
       else
@@ -142,6 +143,10 @@ class RemoveStatusService < BaseService
     redis.publish('timeline:public:local:media', @payload) if @status.local?
   end
 
+  def remove_from_spam_check
+    redis.zremrangebyscore("spam_check:#{@status.account_id}", @status.id, @status.id)
+  end
+
   def lock_options
     { redis: Redis.current, key: "distribute:#{@status.id}" }
   end
diff --git a/config/locales/en.yml b/config/locales/en.yml
index 00b7d1dbe..89251ad40 100644
--- a/config/locales/en.yml
+++ b/config/locales/en.yml
@@ -875,6 +875,8 @@ en:
     profile: Profile
     relationships: Follows and followers
     two_factor_authentication: Two-factor Auth
+  spam_check:
+    spam_detected_and_silenced: This is an automated report. Spam has been detected and the sender has been silenced automatically. If this is a mistake, please unsilence the account.
   statuses:
     attached:
       description: 'Attached: %{attached}'
diff --git a/db/migrate/20190701022101_add_trust_level_to_accounts.rb b/db/migrate/20190701022101_add_trust_level_to_accounts.rb
new file mode 100644
index 000000000..917486d2e
--- /dev/null
+++ b/db/migrate/20190701022101_add_trust_level_to_accounts.rb
@@ -0,0 +1,5 @@
+class AddTrustLevelToAccounts < ActiveRecord::Migration[5.2]
+  def change
+    add_column :accounts, :trust_level, :integer
+  end
+end
diff --git a/db/schema.rb b/db/schema.rb
index 2e38fb1f2..c7b6b9be6 100644
--- a/db/schema.rb
+++ b/db/schema.rb
@@ -148,6 +148,7 @@ ActiveRecord::Schema.define(version: 2019_07_06_233204) do
     t.string "also_known_as", array: true
     t.datetime "silenced_at"
     t.datetime "suspended_at"
+    t.integer "trust_level"
     t.index "(((setweight(to_tsvector('simple'::regconfig, (display_name)::text), 'A'::\"char\") || setweight(to_tsvector('simple'::regconfig, (username)::text), 'B'::\"char\")) || setweight(to_tsvector('simple'::regconfig, (COALESCE(domain, ''::character varying))::text), 'C'::\"char\")))", name: "search_index", using: :gin
     t.index "lower((username)::text), lower((domain)::text)", name: "index_accounts_on_username_and_domain_lower", unique: true
     t.index ["moved_to_account_id"], name: "index_accounts_on_moved_to_account_id"
diff --git a/spec/lib/spam_check_spec.rb b/spec/lib/spam_check_spec.rb
new file mode 100644
index 000000000..c722dc642
--- /dev/null
+++ b/spec/lib/spam_check_spec.rb
@@ -0,0 +1,160 @@
+require 'rails_helper'
+
+RSpec.describe SpamCheck do
+  let!(:sender) { Fabricate(:account) }
+  let!(:alice) { Fabricate(:account, username: 'alice') }
+  let!(:bob) { Fabricate(:account, username: 'bob') }
+
+  def status_with_html(text, options = {})
+    status = PostStatusService.new.call(sender, { text: text }.merge(options))
+    status.update_columns(text: Formatter.instance.format(status), local: false)
+    status
+  end
+
+  describe '#hashable_text' do
+    it 'removes mentions from HTML for remote statuses' do
+      status = status_with_html('@alice Hello')
+      expect(described_class.new(status).hashable_text).to eq 'hello'
+    end
+
+    it 'removes mentions from text for local statuses' do
+      status = PostStatusService.new.call(alice, text: "Hey @#{sender.username}, how are you?")
+      expect(described_class.new(status).hashable_text).to eq 'hey , how are you?'
+    end
+  end
+
+  describe '#insufficient_data?' do
+    it 'returns true when there is no text' do
+      status = status_with_html('@alice')
+      expect(described_class.new(status).insufficient_data?).to be true
+    end
+
+    it 'returns false when there is text' do
+      status = status_with_html('@alice h')
+      expect(described_class.new(status).insufficient_data?).to be false
+    end
+  end
+
+  describe '#digest' do
+    it 'returns a string' do
+      status = status_with_html('@alice Hello world')
+      expect(described_class.new(status).digest).to be_a String
+    end
+  end
+
+  describe '#spam?' do
+    it 'returns false for a unique status' do
+      status = status_with_html('@alice Hello')
+      expect(described_class.new(status).spam?).to be false
+    end
+
+    it 'returns false for different statuses to the same recipient' do
+      status1 = status_with_html('@alice Hello')
+      described_class.new(status1).remember!
+      status2 = status_with_html('@alice Are you available to talk?')
+      expect(described_class.new(status2).spam?).to be false
+    end
+
+    it 'returns false for statuses with different content warnings' do
+      status1 = status_with_html('@alice Are you available to talk?')
+      described_class.new(status1).remember!
+      status2 = status_with_html('@alice Are you available to talk?', spoiler_text: 'This is a completely different matter than what I was talking about previously, I swear!')
+      expect(described_class.new(status2).spam?).to be false
+    end
+
+    it 'returns false for different statuses to different recipients' do
+      status1 = status_with_html('@alice How is it going?')
+      described_class.new(status1).remember!
+      status2 = status_with_html('@bob Are you okay?')
+      expect(described_class.new(status2).spam?).to be false
+    end
+
+    it 'returns false for very short different statuses to different recipients' do
+      status1 = status_with_html('@alice 🙄')
+      described_class.new(status1).remember!
+      status2 = status_with_html('@bob Huh?')
+      expect(described_class.new(status2).spam?).to be false
+    end
+
+    it 'returns false for statuses with no text' do
+      status1 = status_with_html('@alice')
+      described_class.new(status1).remember!
+      status2 = status_with_html('@bob')
+      expect(described_class.new(status2).spam?).to be false
+    end
+
+    it 'returns true for duplicate statuses to the same recipient' do
+      status1 = status_with_html('@alice Hello')
+      described_class.new(status1).remember!
+      status2 = status_with_html('@alice Hello')
+      expect(described_class.new(status2).spam?).to be true
+    end
+
+    it 'returns true for duplicate statuses to different recipients' do
+      status1 = status_with_html('@alice Hello')
+      described_class.new(status1).remember!
+      status2 = status_with_html('@bob Hello')
+      expect(described_class.new(status2).spam?).to be true
+    end
+
+    it 'returns true for nearly identical statuses with random numbers' do
+      source_text = 'Sodium, atomic number 11, was first isolated by Humphry Davy in 1807. A chemical component of salt, he named it Na in honor of the saltiest region on earth, North America.'
+      status1 = status_with_html('@alice ' + source_text + ' 1234')
+      described_class.new(status1).remember!
+      status2 = status_with_html('@bob ' + source_text + ' 9568')
+      expect(described_class.new(status2).spam?).to be true
+    end
+  end
+
+  describe '#skip?' do
+    it 'returns true when the sender is already silenced' do
+      status = status_with_html('@alice Hello')
+      sender.silence!
+      expect(described_class.new(status).skip?).to be true
+    end
+
+    it 'returns true when the mentioned person follows the sender' do
+      status = status_with_html('@alice Hello')
+      alice.follow!(sender)
+      expect(described_class.new(status).skip?).to be true
+    end
+
+    it 'returns false when even one mentioned person doesn\'t follow the sender' do
+      status = status_with_html('@alice @bob Hello')
+      alice.follow!(sender)
+      expect(described_class.new(status).skip?).to be false
+    end
+
+    it 'returns true when the sender is replying to a status that mentions the sender' do
+      parent = PostStatusService.new.call(alice, text: "Hey @#{sender.username}, how are you?")
+      status = status_with_html('@alice @bob Hello', thread: parent)
+      expect(described_class.new(status).skip?).to be true
+    end
+  end
+
+  describe '#remember!' do
+    pending
+  end
+
+  describe '#flag!' do
+    let!(:status1) { status_with_html('@alice General Kenobi you are a bold one') }
+    let!(:status2) { status_with_html('@alice @bob General Kenobi, you are a bold one') }
+
+    before do
+      described_class.new(status1).remember!
+      described_class.new(status2).flag!
+    end
+
+    it 'silences the account' do
+      expect(sender.silenced?).to be true
+    end
+
+    it 'creates a report about the account' do
+      expect(sender.targeted_reports.unresolved.count).to eq 1
+    end
+
+    it 'attaches both matching statuses to the report' do
+      expect(sender.targeted_reports.first.status_ids).to include(status1.id, status2.id)
+    end
+  end
+end
-- 
cgit 


From 9f67e3b9cccb49efd3f3728e0d1a5f68378a11c5 Mon Sep 17 00:00:00 2001
From: ThibG <thib@sitedethib.com>
Date: Mon, 8 Jul 2019 18:17:22 +0200
Subject: Fix Status.remote scope matching *all* statuses (#11265)

---
 app/models/status.rb | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

(limited to 'app/models')

diff --git a/app/models/status.rb b/app/models/status.rb
index 5ddce72de..5adccb722 100644
--- a/app/models/status.rb
+++ b/app/models/status.rb
@@ -82,7 +82,7 @@ class Status < ApplicationRecord
   default_scope { recent }
 
   scope :recent, -> { reorder(id: :desc) }
-  scope :remote, -> { where(local: false).or(where.not(uri: nil)) }
+  scope :remote, -> { where(local: false).where.not(uri: nil) }
   scope :local,  -> { where(local: true).or(where(uri: nil)) }
 
   scope :without_replies, -> { where('statuses.reply = FALSE OR statuses.in_reply_to_account_id = statuses.account_id') }
-- 
cgit 


From 9b1d3e4acb8190509d98ee4ef2ab9b24b45362ee Mon Sep 17 00:00:00 2001
From: Eugen Rochko <eugen@zeonfederated.com>
Date: Tue, 16 Jul 2019 06:30:47 +0200
Subject: Add option to disable real-time updates in web UI (#9984)

Fix #9031
Fix #7913
---
 app/controllers/settings/preferences_controller.rb |  1 +
 app/javascript/mastodon/actions/notifications.js   | 30 ++++++++++++---
 app/javascript/mastodon/actions/timelines.js       | 43 +++++++++++++++-------
 app/javascript/mastodon/compare_id.js              |  5 ++-
 app/javascript/mastodon/components/load_pending.js | 22 +++++++++++
 .../mastodon/components/scrollable_list.js         | 13 ++++++-
 .../components/column_settings.js                  |  2 +-
 .../notifications/components/setting_toggle.js     |  5 ++-
 .../mastodon/features/notifications/index.js       | 12 +++++-
 .../ui/containers/status_list_container.js         |  5 ++-
 app/javascript/mastodon/initial_state.js           |  1 +
 app/javascript/mastodon/reducers/notifications.js  | 30 ++++++++++-----
 app/javascript/mastodon/reducers/settings.js       |  6 ---
 app/javascript/mastodon/reducers/timelines.js      | 40 +++++++++++++-------
 app/lib/user_settings_decorator.rb                 |  5 +++
 app/models/user.rb                                 |  2 +-
 app/serializers/initial_state_serializer.rb        | 23 ++++++------
 .../settings/preferences/appearance/show.html.haml |  3 ++
 config/locales/simple_form.en.yml                  |  2 +
 config/settings.yml                                |  1 +
 20 files changed, 181 insertions(+), 70 deletions(-)
 create mode 100644 app/javascript/mastodon/components/load_pending.js

(limited to 'app/models')

diff --git a/app/controllers/settings/preferences_controller.rb b/app/controllers/settings/preferences_controller.rb
index 0a5c14cca..742c97cdb 100644
--- a/app/controllers/settings/preferences_controller.rb
+++ b/app/controllers/settings/preferences_controller.rb
@@ -55,6 +55,7 @@ class Settings::PreferencesController < Settings::BaseController
       :setting_show_application,
       :setting_advanced_layout,
       :setting_use_blurhash,
+      :setting_use_pending_items,
       notification_emails: %i(follow follow_request reblog favourite mention digest report pending_account),
       interactions: %i(must_be_follower must_be_following must_be_following_dm)
     )
diff --git a/app/javascript/mastodon/actions/notifications.js b/app/javascript/mastodon/actions/notifications.js
index 56c952cb0..d92d972bc 100644
--- a/app/javascript/mastodon/actions/notifications.js
+++ b/app/javascript/mastodon/actions/notifications.js
@@ -12,6 +12,8 @@ import { defineMessages } from 'react-intl';
 import { List as ImmutableList } from 'immutable';
 import { unescapeHTML } from '../utils/html';
 import { getFiltersRegex } from '../selectors';
+import { usePendingItems as preferPendingItems } from 'mastodon/initial_state';
+import compareId from 'mastodon/compare_id';
 
 export const NOTIFICATIONS_UPDATE      = 'NOTIFICATIONS_UPDATE';
 export const NOTIFICATIONS_UPDATE_NOOP = 'NOTIFICATIONS_UPDATE_NOOP';
@@ -22,8 +24,9 @@ export const NOTIFICATIONS_EXPAND_FAIL    = 'NOTIFICATIONS_EXPAND_FAIL';
 
 export const NOTIFICATIONS_FILTER_SET = 'NOTIFICATIONS_FILTER_SET';
 
-export const NOTIFICATIONS_CLEAR      = 'NOTIFICATIONS_CLEAR';
-export const NOTIFICATIONS_SCROLL_TOP = 'NOTIFICATIONS_SCROLL_TOP';
+export const NOTIFICATIONS_CLEAR        = 'NOTIFICATIONS_CLEAR';
+export const NOTIFICATIONS_SCROLL_TOP   = 'NOTIFICATIONS_SCROLL_TOP';
+export const NOTIFICATIONS_LOAD_PENDING = 'NOTIFICATIONS_LOAD_PENDING';
 
 defineMessages({
   mention: { id: 'notification.mention', defaultMessage: '{name} mentioned you' },
@@ -38,6 +41,10 @@ const fetchRelatedRelationships = (dispatch, notifications) => {
   }
 };
 
+export const loadPending = () => ({
+  type: NOTIFICATIONS_LOAD_PENDING,
+});
+
 export function updateNotifications(notification, intlMessages, intlLocale) {
   return (dispatch, getState) => {
     const showInColumn = getState().getIn(['settings', 'notifications', 'shows', notification.type], true);
@@ -69,6 +76,7 @@ export function updateNotifications(notification, intlMessages, intlLocale) {
       dispatch({
         type: NOTIFICATIONS_UPDATE,
         notification,
+        usePendingItems: preferPendingItems,
         meta: (playSound && !filtered) ? { sound: 'boop' } : undefined,
       });
 
@@ -122,10 +130,19 @@ export function expandNotifications({ maxId } = {}, done = noOp) {
         : excludeTypesFromFilter(activeFilter),
     };
 
-    if (!maxId && notifications.get('items').size > 0) {
-      params.since_id = notifications.getIn(['items', 0, 'id']);
+    if (!params.max_id && (notifications.get('items', ImmutableList()).size + notifications.get('pendingItems', ImmutableList()).size) > 0) {
+      const a = notifications.getIn(['pendingItems', 0, 'id']);
+      const b = notifications.getIn(['items', 0, 'id']);
+
+      if (a && b && compareId(a, b) > 0) {
+        params.since_id = a;
+      } else {
+        params.since_id = b || a;
+      }
     }
 
+    const isLoadingRecent = !!params.since_id;
+
     dispatch(expandNotificationsRequest(isLoadingMore));
 
     api(getState).get('/api/v1/notifications', { params }).then(response => {
@@ -134,7 +151,7 @@ export function expandNotifications({ maxId } = {}, done = noOp) {
       dispatch(importFetchedAccounts(response.data.map(item => item.account)));
       dispatch(importFetchedStatuses(response.data.map(item => item.status).filter(status => !!status)));
 
-      dispatch(expandNotificationsSuccess(response.data, next ? next.uri : null, isLoadingMore));
+      dispatch(expandNotificationsSuccess(response.data, next ? next.uri : null, isLoadingMore, isLoadingRecent && preferPendingItems));
       fetchRelatedRelationships(dispatch, response.data);
       done();
     }).catch(error => {
@@ -151,11 +168,12 @@ export function expandNotificationsRequest(isLoadingMore) {
   };
 };
 
-export function expandNotificationsSuccess(notifications, next, isLoadingMore) {
+export function expandNotificationsSuccess(notifications, next, isLoadingMore, usePendingItems) {
   return {
     type: NOTIFICATIONS_EXPAND_SUCCESS,
     notifications,
     next,
+    usePendingItems,
     skipLoading: !isLoadingMore,
   };
 };
diff --git a/app/javascript/mastodon/actions/timelines.js b/app/javascript/mastodon/actions/timelines.js
index 06c21b96b..7eeba2aa7 100644
--- a/app/javascript/mastodon/actions/timelines.js
+++ b/app/javascript/mastodon/actions/timelines.js
@@ -1,6 +1,8 @@
 import { importFetchedStatus, importFetchedStatuses } from './importer';
-import api, { getLinks } from '../api';
+import api, { getLinks } from 'mastodon/api';
 import { Map as ImmutableMap, List as ImmutableList } from 'immutable';
+import compareId from 'mastodon/compare_id';
+import { usePendingItems as preferPendingItems } from 'mastodon/initial_state';
 
 export const TIMELINE_UPDATE  = 'TIMELINE_UPDATE';
 export const TIMELINE_DELETE  = 'TIMELINE_DELETE';
@@ -10,10 +12,15 @@ export const TIMELINE_EXPAND_REQUEST = 'TIMELINE_EXPAND_REQUEST';
 export const TIMELINE_EXPAND_SUCCESS = 'TIMELINE_EXPAND_SUCCESS';
 export const TIMELINE_EXPAND_FAIL    = 'TIMELINE_EXPAND_FAIL';
 
-export const TIMELINE_SCROLL_TOP = 'TIMELINE_SCROLL_TOP';
+export const TIMELINE_SCROLL_TOP   = 'TIMELINE_SCROLL_TOP';
+export const TIMELINE_LOAD_PENDING = 'TIMELINE_LOAD_PENDING';
+export const TIMELINE_DISCONNECT   = 'TIMELINE_DISCONNECT';
+export const TIMELINE_CONNECT      = 'TIMELINE_CONNECT';
 
-export const TIMELINE_CONNECT    = 'TIMELINE_CONNECT';
-export const TIMELINE_DISCONNECT = 'TIMELINE_DISCONNECT';
+export const loadPending = timeline => ({
+  type: TIMELINE_LOAD_PENDING,
+  timeline,
+});
 
 export function updateTimeline(timeline, status, accept) {
   return dispatch => {
@@ -27,6 +34,7 @@ export function updateTimeline(timeline, status, accept) {
       type: TIMELINE_UPDATE,
       timeline,
       status,
+      usePendingItems: preferPendingItems,
     });
   };
 };
@@ -71,8 +79,15 @@ export function expandTimeline(timelineId, path, params = {}, done = noOp) {
       return;
     }
 
-    if (!params.max_id && !params.pinned && timeline.get('items', ImmutableList()).size > 0) {
-      params.since_id = timeline.getIn(['items', 0]);
+    if (!params.max_id && !params.pinned && (timeline.get('items', ImmutableList()).size + timeline.get('pendingItems', ImmutableList()).size) > 0) {
+      const a = timeline.getIn(['pendingItems', 0]);
+      const b = timeline.getIn(['items', 0]);
+
+      if (a && b && compareId(a, b) > 0) {
+        params.since_id = a;
+      } else {
+        params.since_id = b || a;
+      }
     }
 
     const isLoadingRecent = !!params.since_id;
@@ -82,7 +97,7 @@ export function expandTimeline(timelineId, path, params = {}, done = noOp) {
     api(getState).get(path, { params }).then(response => {
       const next = getLinks(response).refs.find(link => link.rel === 'next');
       dispatch(importFetchedStatuses(response.data));
-      dispatch(expandTimelineSuccess(timelineId, response.data, next ? next.uri : null, response.code === 206, isLoadingRecent, isLoadingMore));
+      dispatch(expandTimelineSuccess(timelineId, response.data, next ? next.uri : null, response.code === 206, isLoadingRecent, isLoadingMore, isLoadingRecent && preferPendingItems));
       done();
     }).catch(error => {
       dispatch(expandTimelineFail(timelineId, error, isLoadingMore));
@@ -115,7 +130,7 @@ export function expandTimelineRequest(timeline, isLoadingMore) {
   };
 };
 
-export function expandTimelineSuccess(timeline, statuses, next, partial, isLoadingRecent, isLoadingMore) {
+export function expandTimelineSuccess(timeline, statuses, next, partial, isLoadingRecent, isLoadingMore, usePendingItems) {
   return {
     type: TIMELINE_EXPAND_SUCCESS,
     timeline,
@@ -123,6 +138,7 @@ export function expandTimelineSuccess(timeline, statuses, next, partial, isLoadi
     next,
     partial,
     isLoadingRecent,
+    usePendingItems,
     skipLoading: !isLoadingMore,
   };
 };
@@ -151,9 +167,8 @@ export function connectTimeline(timeline) {
   };
 };
 
-export function disconnectTimeline(timeline) {
-  return {
-    type: TIMELINE_DISCONNECT,
-    timeline,
-  };
-};
+export const disconnectTimeline = timeline => ({
+  type: TIMELINE_DISCONNECT,
+  timeline,
+  usePendingItems: preferPendingItems,
+});
diff --git a/app/javascript/mastodon/compare_id.js b/app/javascript/mastodon/compare_id.js
index aaff66481..66cf51c4b 100644
--- a/app/javascript/mastodon/compare_id.js
+++ b/app/javascript/mastodon/compare_id.js
@@ -1,10 +1,11 @@
-export default function compareId(id1, id2) {
+export default function compareId (id1, id2) {
   if (id1 === id2) {
     return 0;
   }
+
   if (id1.length === id2.length) {
     return id1 > id2 ? 1 : -1;
   } else {
     return id1.length > id2.length ? 1 : -1;
   }
-}
+};
diff --git a/app/javascript/mastodon/components/load_pending.js b/app/javascript/mastodon/components/load_pending.js
new file mode 100644
index 000000000..7e2702403
--- /dev/null
+++ b/app/javascript/mastodon/components/load_pending.js
@@ -0,0 +1,22 @@
+import React from 'react';
+import { FormattedMessage } from 'react-intl';
+import PropTypes from 'prop-types';
+
+export default class LoadPending extends React.PureComponent {
+
+  static propTypes = {
+    onClick: PropTypes.func,
+    count: PropTypes.number,
+  }
+
+  render() {
+    const { count } = this.props;
+
+    return (
+      <button className='load-more load-gap' onClick={this.props.onClick}>
+        <FormattedMessage id='load_pending' defaultMessage='{count, plural, one {# new item} other {# new items}}' values={{ count }} />
+      </button>
+    );
+  }
+
+}
diff --git a/app/javascript/mastodon/components/scrollable_list.js b/app/javascript/mastodon/components/scrollable_list.js
index 0376cf85a..553c16352 100644
--- a/app/javascript/mastodon/components/scrollable_list.js
+++ b/app/javascript/mastodon/components/scrollable_list.js
@@ -3,6 +3,7 @@ import { ScrollContainer } from 'react-router-scroll-4';
 import PropTypes from 'prop-types';
 import IntersectionObserverArticleContainer from '../containers/intersection_observer_article_container';
 import LoadMore from './load_more';
+import LoadPending from './load_pending';
 import IntersectionObserverWrapper from '../features/ui/util/intersection_observer_wrapper';
 import { throttle } from 'lodash';
 import { List as ImmutableList } from 'immutable';
@@ -21,6 +22,7 @@ export default class ScrollableList extends PureComponent {
   static propTypes = {
     scrollKey: PropTypes.string.isRequired,
     onLoadMore: PropTypes.func,
+    onLoadPending: PropTypes.func,
     onScrollToTop: PropTypes.func,
     onScroll: PropTypes.func,
     trackScroll: PropTypes.bool,
@@ -28,6 +30,7 @@ export default class ScrollableList extends PureComponent {
     isLoading: PropTypes.bool,
     showLoading: PropTypes.bool,
     hasMore: PropTypes.bool,
+    numPending: PropTypes.number,
     prepend: PropTypes.node,
     alwaysPrepend: PropTypes.bool,
     emptyMessage: PropTypes.node,
@@ -225,12 +228,18 @@ export default class ScrollableList extends PureComponent {
     this.props.onLoadMore();
   }
 
+  handleLoadPending = e => {
+    e.preventDefault();
+    this.props.onLoadPending();
+  }
+
   render () {
-    const { children, scrollKey, trackScroll, shouldUpdateScroll, showLoading, isLoading, hasMore, prepend, alwaysPrepend, emptyMessage, onLoadMore } = this.props;
+    const { children, scrollKey, trackScroll, shouldUpdateScroll, showLoading, isLoading, hasMore, numPending, prepend, alwaysPrepend, emptyMessage, onLoadMore } = this.props;
     const { fullscreen } = this.state;
     const childrenCount = React.Children.count(children);
 
     const loadMore     = (hasMore && onLoadMore) ? <LoadMore visible={!isLoading} onClick={this.handleLoadMore} /> : null;
+    const loadPending  = (numPending > 0) ? <LoadPending count={numPending} onClick={this.handleLoadPending} /> : null;
     let scrollableArea = null;
 
     if (showLoading) {
@@ -251,6 +260,8 @@ export default class ScrollableList extends PureComponent {
           <div role='feed' className='item-list'>
             {prepend}
 
+            {loadPending}
+
             {React.Children.map(this.props.children, (child, index) => (
               <IntersectionObserverArticleContainer
                 key={child.key}
diff --git a/app/javascript/mastodon/features/community_timeline/components/column_settings.js b/app/javascript/mastodon/features/community_timeline/components/column_settings.js
index 8250190a7..0cb6db883 100644
--- a/app/javascript/mastodon/features/community_timeline/components/column_settings.js
+++ b/app/javascript/mastodon/features/community_timeline/components/column_settings.js
@@ -20,7 +20,7 @@ class ColumnSettings extends React.PureComponent {
     return (
       <div>
         <div className='column-settings__row'>
-          <SettingToggle settings={settings} settingPath={['other', 'onlyMedia']} onChange={onChange} label={<FormattedMessage id='community.column_settings.media_only' defaultMessage='Media Only' />} />
+          <SettingToggle settings={settings} settingPath={['other', 'onlyMedia']} onChange={onChange} label={<FormattedMessage id='community.column_settings.media_only' defaultMessage='Media only' />} />
         </div>
       </div>
     );
diff --git a/app/javascript/mastodon/features/notifications/components/setting_toggle.js b/app/javascript/mastodon/features/notifications/components/setting_toggle.js
index 7aec16d2e..e6f593ef8 100644
--- a/app/javascript/mastodon/features/notifications/components/setting_toggle.js
+++ b/app/javascript/mastodon/features/notifications/components/setting_toggle.js
@@ -11,6 +11,7 @@ export default class SettingToggle extends React.PureComponent {
     settingPath: PropTypes.array.isRequired,
     label: PropTypes.node.isRequired,
     onChange: PropTypes.func.isRequired,
+    defaultValue: PropTypes.bool,
   }
 
   onChange = ({ target }) => {
@@ -18,12 +19,12 @@ export default class SettingToggle extends React.PureComponent {
   }
 
   render () {
-    const { prefix, settings, settingPath, label } = this.props;
+    const { prefix, settings, settingPath, label, defaultValue } = this.props;
     const id = ['setting-toggle', prefix, ...settingPath].filter(Boolean).join('-');
 
     return (
       <div className='setting-toggle'>
-        <Toggle id={id} checked={settings.getIn(settingPath)} onChange={this.onChange} onKeyDown={this.onKeyDown} />
+        <Toggle id={id} checked={settings.getIn(settingPath, defaultValue)} onChange={this.onChange} onKeyDown={this.onKeyDown} />
         <label htmlFor={id} className='setting-toggle__label'>{label}</label>
       </div>
     );
diff --git a/app/javascript/mastodon/features/notifications/index.js b/app/javascript/mastodon/features/notifications/index.js
index 006c45657..df4ad6f2a 100644
--- a/app/javascript/mastodon/features/notifications/index.js
+++ b/app/javascript/mastodon/features/notifications/index.js
@@ -4,7 +4,7 @@ import PropTypes from 'prop-types';
 import ImmutablePropTypes from 'react-immutable-proptypes';
 import Column from '../../components/column';
 import ColumnHeader from '../../components/column_header';
-import { expandNotifications, scrollTopNotifications } from '../../actions/notifications';
+import { expandNotifications, scrollTopNotifications, loadPending } from '../../actions/notifications';
 import { addColumn, removeColumn, moveColumn } from '../../actions/columns';
 import NotificationContainer from './containers/notification_container';
 import { defineMessages, injectIntl, FormattedMessage } from 'react-intl';
@@ -41,6 +41,7 @@ const mapStateToProps = state => ({
   isLoading: state.getIn(['notifications', 'isLoading'], true),
   isUnread: state.getIn(['notifications', 'unread']) > 0,
   hasMore: state.getIn(['notifications', 'hasMore']),
+  numPending: state.getIn(['notifications', 'pendingItems'], ImmutableList()).size,
 });
 
 export default @connect(mapStateToProps)
@@ -58,6 +59,7 @@ class Notifications extends React.PureComponent {
     isUnread: PropTypes.bool,
     multiColumn: PropTypes.bool,
     hasMore: PropTypes.bool,
+    numPending: PropTypes.number,
   };
 
   static defaultProps = {
@@ -80,6 +82,10 @@ class Notifications extends React.PureComponent {
     this.props.dispatch(expandNotifications({ maxId: last && last.get('id') }));
   }, 300, { leading: true });
 
+  handleLoadPending = () => {
+    this.props.dispatch(loadPending());
+  };
+
   handleScrollToTop = debounce(() => {
     this.props.dispatch(scrollTopNotifications(true));
   }, 100);
@@ -136,7 +142,7 @@ class Notifications extends React.PureComponent {
   }
 
   render () {
-    const { intl, notifications, shouldUpdateScroll, isLoading, isUnread, columnId, multiColumn, hasMore, showFilterBar } = this.props;
+    const { intl, notifications, shouldUpdateScroll, isLoading, isUnread, columnId, multiColumn, hasMore, numPending, showFilterBar } = this.props;
     const pinned = !!columnId;
     const emptyMessage = <FormattedMessage id='empty_column.notifications' defaultMessage="You don't have any notifications yet. Interact with others to start the conversation." />;
 
@@ -178,8 +184,10 @@ class Notifications extends React.PureComponent {
         isLoading={isLoading}
         showLoading={isLoading && notifications.size === 0}
         hasMore={hasMore}
+        numPending={numPending}
         emptyMessage={emptyMessage}
         onLoadMore={this.handleLoadOlder}
+        onLoadPending={this.handleLoadPending}
         onScrollToTop={this.handleScrollToTop}
         onScroll={this.handleScroll}
         shouldUpdateScroll={shouldUpdateScroll}
diff --git a/app/javascript/mastodon/features/ui/containers/status_list_container.js b/app/javascript/mastodon/features/ui/containers/status_list_container.js
index 3df5b7bea..7b8eb652b 100644
--- a/app/javascript/mastodon/features/ui/containers/status_list_container.js
+++ b/app/javascript/mastodon/features/ui/containers/status_list_container.js
@@ -1,6 +1,6 @@
 import { connect } from 'react-redux';
 import StatusList from '../../../components/status_list';
-import { scrollTopTimeline } from '../../../actions/timelines';
+import { scrollTopTimeline, loadPending } from '../../../actions/timelines';
 import { Map as ImmutableMap, List as ImmutableList } from 'immutable';
 import { createSelector } from 'reselect';
 import { debounce } from 'lodash';
@@ -37,6 +37,7 @@ const makeMapStateToProps = () => {
     isLoading: state.getIn(['timelines', timelineId, 'isLoading'], true),
     isPartial: state.getIn(['timelines', timelineId, 'isPartial'], false),
     hasMore:   state.getIn(['timelines', timelineId, 'hasMore']),
+    numPending: state.getIn(['timelines', timelineId, 'pendingItems'], ImmutableList()).size,
   });
 
   return mapStateToProps;
@@ -52,6 +53,8 @@ const mapDispatchToProps = (dispatch, { timelineId }) => ({
     dispatch(scrollTopTimeline(timelineId, false));
   }, 100),
 
+  onLoadPending: () => dispatch(loadPending(timelineId)),
+
 });
 
 export default connect(makeMapStateToProps, mapDispatchToProps)(StatusList);
diff --git a/app/javascript/mastodon/initial_state.js b/app/javascript/mastodon/initial_state.js
index f0f131ff5..cb2ccc7c4 100644
--- a/app/javascript/mastodon/initial_state.js
+++ b/app/javascript/mastodon/initial_state.js
@@ -21,5 +21,6 @@ export const profile_directory = getMeta('profile_directory');
 export const isStaff = getMeta('is_staff');
 export const forceSingleColumn = !getMeta('advanced_layout');
 export const useBlurhash = getMeta('use_blurhash');
+export const usePendingItems = getMeta('use_pending_items');
 
 export default initialState;
diff --git a/app/javascript/mastodon/reducers/notifications.js b/app/javascript/mastodon/reducers/notifications.js
index 4d9604de9..e94a4946b 100644
--- a/app/javascript/mastodon/reducers/notifications.js
+++ b/app/javascript/mastodon/reducers/notifications.js
@@ -6,6 +6,7 @@ import {
   NOTIFICATIONS_FILTER_SET,
   NOTIFICATIONS_CLEAR,
   NOTIFICATIONS_SCROLL_TOP,
+  NOTIFICATIONS_LOAD_PENDING,
 } from '../actions/notifications';
 import {
   ACCOUNT_BLOCK_SUCCESS,
@@ -16,6 +17,7 @@ import { Map as ImmutableMap, List as ImmutableList } from 'immutable';
 import compareId from '../compare_id';
 
 const initialState = ImmutableMap({
+  pendingItems: ImmutableList(),
   items: ImmutableList(),
   hasMore: true,
   top: false,
@@ -31,7 +33,11 @@ const notificationToMap = notification => ImmutableMap({
   status: notification.status ? notification.status.id : null,
 });
 
-const normalizeNotification = (state, notification) => {
+const normalizeNotification = (state, notification, usePendingItems) => {
+  if (usePendingItems) {
+    return state.update('pendingItems', list => list.unshift(notificationToMap(notification)));
+  }
+
   const top = state.get('top');
 
   if (!top) {
@@ -47,7 +53,7 @@ const normalizeNotification = (state, notification) => {
   });
 };
 
-const expandNormalizedNotifications = (state, notifications, next) => {
+const expandNormalizedNotifications = (state, notifications, next, usePendingItems) => {
   let items = ImmutableList();
 
   notifications.forEach((n, i) => {
@@ -56,7 +62,7 @@ const expandNormalizedNotifications = (state, notifications, next) => {
 
   return state.withMutations(mutable => {
     if (!items.isEmpty()) {
-      mutable.update('items', list => {
+      mutable.update(usePendingItems ? 'pendingItems' : 'items', list => {
         const lastIndex = 1 + list.findLastIndex(
           item => item !== null && (compareId(item.get('id'), items.last().get('id')) > 0 || item.get('id') === items.last().get('id'))
         );
@@ -78,7 +84,8 @@ const expandNormalizedNotifications = (state, notifications, next) => {
 };
 
 const filterNotifications = (state, relationship) => {
-  return state.update('items', list => list.filterNot(item => item !== null && item.get('account') === relationship.id));
+  const helper = list => list.filterNot(item => item !== null && item.get('account') === relationship.id);
+  return state.update('items', helper).update('pendingItems', helper);
 };
 
 const updateTop = (state, top) => {
@@ -90,34 +97,37 @@ const updateTop = (state, top) => {
 };
 
 const deleteByStatus = (state, statusId) => {
-  return state.update('items', list => list.filterNot(item => item !== null && item.get('status') === statusId));
+  const helper = list => list.filterNot(item => item !== null && item.get('status') === statusId);
+  return state.update('items', helper).update('pendingItems', helper);
 };
 
 export default function notifications(state = initialState, action) {
   switch(action.type) {
+  case NOTIFICATIONS_LOAD_PENDING:
+    return state.update('items', list => state.get('pendingItems').concat(list.take(40))).set('pendingItems', ImmutableList()).set('unread', 0);
   case NOTIFICATIONS_EXPAND_REQUEST:
     return state.set('isLoading', true);
   case NOTIFICATIONS_EXPAND_FAIL:
     return state.set('isLoading', false);
   case NOTIFICATIONS_FILTER_SET:
-    return state.set('items', ImmutableList()).set('hasMore', true);
+    return state.set('items', ImmutableList()).set('pendingItems', ImmutableList()).set('hasMore', true);
   case NOTIFICATIONS_SCROLL_TOP:
     return updateTop(state, action.top);
   case NOTIFICATIONS_UPDATE:
-    return normalizeNotification(state, action.notification);
+    return normalizeNotification(state, action.notification, action.usePendingItems);
   case NOTIFICATIONS_EXPAND_SUCCESS:
-    return expandNormalizedNotifications(state, action.notifications, action.next);
+    return expandNormalizedNotifications(state, action.notifications, action.next, action.usePendingItems);
   case ACCOUNT_BLOCK_SUCCESS:
     return filterNotifications(state, action.relationship);
   case ACCOUNT_MUTE_SUCCESS:
     return action.relationship.muting_notifications ? filterNotifications(state, action.relationship) : state;
   case NOTIFICATIONS_CLEAR:
-    return state.set('items', ImmutableList()).set('hasMore', false);
+    return state.set('items', ImmutableList()).set('pendingItems', ImmutableList()).set('hasMore', false);
   case TIMELINE_DELETE:
     return deleteByStatus(state, action.id);
   case TIMELINE_DISCONNECT:
     return action.timeline === 'home' ?
-      state.update('items', items => items.first() ? items.unshift(null) : items) :
+      state.update(action.usePendingItems ? 'pendingItems' : 'items', items => items.first() ? items.unshift(null) : items) :
       state;
   default:
     return state;
diff --git a/app/javascript/mastodon/reducers/settings.js b/app/javascript/mastodon/reducers/settings.js
index a0eea137f..033bfc999 100644
--- a/app/javascript/mastodon/reducers/settings.js
+++ b/app/javascript/mastodon/reducers/settings.js
@@ -10,8 +10,6 @@ import uuid from '../uuid';
 const initialState = ImmutableMap({
   saved: true,
 
-  onboarded: false,
-
   skinTone: 1,
 
   home: ImmutableMap({
@@ -74,10 +72,6 @@ const initialState = ImmutableMap({
       body: '',
     }),
   }),
-
-  trends: ImmutableMap({
-    show: true,
-  }),
 });
 
 const defaultColumns = fromJS([
diff --git a/app/javascript/mastodon/reducers/timelines.js b/app/javascript/mastodon/reducers/timelines.js
index 309a95a19..0b036f5fe 100644
--- a/app/javascript/mastodon/reducers/timelines.js
+++ b/app/javascript/mastodon/reducers/timelines.js
@@ -8,6 +8,7 @@ import {
   TIMELINE_SCROLL_TOP,
   TIMELINE_CONNECT,
   TIMELINE_DISCONNECT,
+  TIMELINE_LOAD_PENDING,
 } from '../actions/timelines';
 import {
   ACCOUNT_BLOCK_SUCCESS,
@@ -25,10 +26,11 @@ const initialTimeline = ImmutableMap({
   top: true,
   isLoading: false,
   hasMore: true,
+  pendingItems: ImmutableList(),
   items: ImmutableList(),
 });
 
-const expandNormalizedTimeline = (state, timeline, statuses, next, isPartial, isLoadingRecent) => {
+const expandNormalizedTimeline = (state, timeline, statuses, next, isPartial, isLoadingRecent, usePendingItems) => {
   return state.update(timeline, initialTimeline, map => map.withMutations(mMap => {
     mMap.set('isLoading', false);
     mMap.set('isPartial', isPartial);
@@ -38,7 +40,7 @@ const expandNormalizedTimeline = (state, timeline, statuses, next, isPartial, is
     if (timeline.endsWith(':pinned')) {
       mMap.set('items', statuses.map(status => status.get('id')));
     } else if (!statuses.isEmpty()) {
-      mMap.update('items', ImmutableList(), oldIds => {
+      mMap.update(usePendingItems ? 'pendingItems' : 'items', ImmutableList(), oldIds => {
         const newIds = statuses.map(status => status.get('id'));
 
         const lastIndex = oldIds.findLastIndex(id => id !== null && compareId(id, newIds.last()) >= 0) + 1;
@@ -57,7 +59,15 @@ const expandNormalizedTimeline = (state, timeline, statuses, next, isPartial, is
   }));
 };
 
-const updateTimeline = (state, timeline, status) => {
+const updateTimeline = (state, timeline, status, usePendingItems) => {
+  if (usePendingItems) {
+    if (state.getIn([timeline, 'pendingItems'], ImmutableList()).includes(status.get('id')) || state.getIn([timeline, 'items'], ImmutableList()).includes(status.get('id'))) {
+      return state;
+    }
+
+    return state.update(timeline, initialTimeline, map => map.update('pendingItems', list => list.unshift(status.get('id'))));
+  }
+
   const top        = state.getIn([timeline, 'top']);
   const ids        = state.getIn([timeline, 'items'], ImmutableList());
   const includesId = ids.includes(status.get('id'));
@@ -78,8 +88,10 @@ const updateTimeline = (state, timeline, status) => {
 
 const deleteStatus = (state, id, accountId, references, exclude_account = null) => {
   state.keySeq().forEach(timeline => {
-    if (exclude_account === null || (timeline !== `account:${exclude_account}` && !timeline.startsWith(`account:${exclude_account}:`)))
-      state = state.updateIn([timeline, 'items'], list => list.filterNot(item => item === id));
+    if (exclude_account === null || (timeline !== `account:${exclude_account}` && !timeline.startsWith(`account:${exclude_account}:`))) {
+      const helper = list => list.filterNot(item => item === id);
+      state = state.updateIn([timeline, 'items'], helper).updateIn([timeline, 'pendingItems'], helper);
+    }
   });
 
   // Remove reblogs of deleted status
@@ -109,11 +121,10 @@ const filterTimelines = (state, relationship, statuses) => {
   return state;
 };
 
-const filterTimeline = (timeline, state, relationship, statuses) =>
-  state.updateIn([timeline, 'items'], ImmutableList(), list =>
-    list.filterNot(statusId =>
-      statuses.getIn([statusId, 'account']) === relationship.id
-    ));
+const filterTimeline = (timeline, state, relationship, statuses) => {
+  const helper = list => list.filterNot(statusId => statuses.getIn([statusId, 'account']) === relationship.id);
+  return state.updateIn([timeline, 'items'], ImmutableList(), helper).updateIn([timeline, 'pendingItems'], ImmutableList(), helper);
+};
 
 const updateTop = (state, timeline, top) => {
   return state.update(timeline, initialTimeline, map => map.withMutations(mMap => {
@@ -124,14 +135,17 @@ const updateTop = (state, timeline, top) => {
 
 export default function timelines(state = initialState, action) {
   switch(action.type) {
+  case TIMELINE_LOAD_PENDING:
+    return state.update(action.timeline, initialTimeline, map =>
+      map.update('items', list => map.get('pendingItems').concat(list.take(40))).set('pendingItems', ImmutableList()).set('unread', 0));
   case TIMELINE_EXPAND_REQUEST:
     return state.update(action.timeline, initialTimeline, map => map.set('isLoading', true));
   case TIMELINE_EXPAND_FAIL:
     return state.update(action.timeline, initialTimeline, map => map.set('isLoading', false));
   case TIMELINE_EXPAND_SUCCESS:
-    return expandNormalizedTimeline(state, action.timeline, fromJS(action.statuses), action.next, action.partial, action.isLoadingRecent);
+    return expandNormalizedTimeline(state, action.timeline, fromJS(action.statuses), action.next, action.partial, action.isLoadingRecent, action.usePendingItems);
   case TIMELINE_UPDATE:
-    return updateTimeline(state, action.timeline, fromJS(action.status));
+    return updateTimeline(state, action.timeline, fromJS(action.status), action.usePendingItems);
   case TIMELINE_DELETE:
     return deleteStatus(state, action.id, action.accountId, action.references, action.reblogOf);
   case TIMELINE_CLEAR:
@@ -149,7 +163,7 @@ export default function timelines(state = initialState, action) {
     return state.update(
       action.timeline,
       initialTimeline,
-      map => map.set('online', false).update('items', items => items.first() ? items.unshift(null) : items)
+      map => map.set('online', false).update(action.usePendingItems ? 'pendingItems' : 'items', items => items.first() ? items.unshift(null) : items)
     );
   default:
     return state;
diff --git a/app/lib/user_settings_decorator.rb b/app/lib/user_settings_decorator.rb
index aaf95cc19..9ae9986c2 100644
--- a/app/lib/user_settings_decorator.rb
+++ b/app/lib/user_settings_decorator.rb
@@ -35,6 +35,7 @@ class UserSettingsDecorator
     user.settings['show_application']    = show_application_preference if change?('setting_show_application')
     user.settings['advanced_layout']     = advanced_layout_preference if change?('setting_advanced_layout')
     user.settings['use_blurhash']        = use_blurhash_preference if change?('setting_use_blurhash')
+    user.settings['use_pending_items']   = use_pending_items_preference if change?('setting_use_pending_items')
   end
 
   def merged_notification_emails
@@ -117,6 +118,10 @@ class UserSettingsDecorator
     boolean_cast_setting 'setting_use_blurhash'
   end
 
+  def use_pending_items_preference
+    boolean_cast_setting 'setting_use_pending_items'
+  end
+
   def boolean_cast_setting(key)
     ActiveModel::Type::Boolean.new.cast(settings[key])
   end
diff --git a/app/models/user.rb b/app/models/user.rb
index 3a4b415dd..31c99630c 100644
--- a/app/models/user.rb
+++ b/app/models/user.rb
@@ -106,7 +106,7 @@ class User < ApplicationRecord
   delegate :auto_play_gif, :default_sensitive, :unfollow_modal, :boost_modal, :delete_modal,
            :reduce_motion, :system_font_ui, :noindex, :theme, :display_media, :hide_network,
            :expand_spoilers, :default_language, :aggregate_reblogs, :show_application,
-           :advanced_layout, :use_blurhash, to: :settings, prefix: :setting, allow_nil: false
+           :advanced_layout, :use_blurhash, :use_pending_items, to: :settings, prefix: :setting, allow_nil: false
 
   attr_reader :invite_code
   attr_writer :external
diff --git a/app/serializers/initial_state_serializer.rb b/app/serializers/initial_state_serializer.rb
index 54f99d570..7e5d3eda9 100644
--- a/app/serializers/initial_state_serializer.rb
+++ b/app/serializers/initial_state_serializer.rb
@@ -23,17 +23,18 @@ class InitialStateSerializer < ActiveModel::Serializer
     }
 
     if object.current_account
-      store[:me]              = object.current_account.id.to_s
-      store[:unfollow_modal]  = object.current_account.user.setting_unfollow_modal
-      store[:boost_modal]     = object.current_account.user.setting_boost_modal
-      store[:delete_modal]    = object.current_account.user.setting_delete_modal
-      store[:auto_play_gif]   = object.current_account.user.setting_auto_play_gif
-      store[:display_media]   = object.current_account.user.setting_display_media
-      store[:expand_spoilers] = object.current_account.user.setting_expand_spoilers
-      store[:reduce_motion]   = object.current_account.user.setting_reduce_motion
-      store[:advanced_layout] = object.current_account.user.setting_advanced_layout
-      store[:use_blurhash]    = object.current_account.user.setting_use_blurhash
-      store[:is_staff]        = object.current_account.user.staff?
+      store[:me]                = object.current_account.id.to_s
+      store[:unfollow_modal]    = object.current_account.user.setting_unfollow_modal
+      store[:boost_modal]       = object.current_account.user.setting_boost_modal
+      store[:delete_modal]      = object.current_account.user.setting_delete_modal
+      store[:auto_play_gif]     = object.current_account.user.setting_auto_play_gif
+      store[:display_media]     = object.current_account.user.setting_display_media
+      store[:expand_spoilers]   = object.current_account.user.setting_expand_spoilers
+      store[:reduce_motion]     = object.current_account.user.setting_reduce_motion
+      store[:advanced_layout]   = object.current_account.user.setting_advanced_layout
+      store[:use_blurhash]      = object.current_account.user.setting_use_blurhash
+      store[:use_pending_items] = object.current_account.user.setting_use_pending_items
+      store[:is_staff]          = object.current_account.user.staff?
     end
 
     store
diff --git a/app/views/settings/preferences/appearance/show.html.haml b/app/views/settings/preferences/appearance/show.html.haml
index c13882801..e279a61c4 100644
--- a/app/views/settings/preferences/appearance/show.html.haml
+++ b/app/views/settings/preferences/appearance/show.html.haml
@@ -17,6 +17,9 @@
 
   %h4= t 'appearance.animations_and_accessibility'
 
+  .fields-group
+    = f.input :setting_use_pending_items, as: :boolean, wrapper: :with_label
+
   .fields-group
     = f.input :setting_auto_play_gif, as: :boolean, wrapper: :with_label, recommended: true
     = f.input :setting_reduce_motion, as: :boolean, wrapper: :with_label
diff --git a/config/locales/simple_form.en.yml b/config/locales/simple_form.en.yml
index 75cbec9de..12a7ec2b3 100644
--- a/config/locales/simple_form.en.yml
+++ b/config/locales/simple_form.en.yml
@@ -35,6 +35,7 @@ en:
         setting_noindex: Affects your public profile and status pages
         setting_show_application: The application you use to toot will be displayed in the detailed view of your toots
         setting_use_blurhash: Gradients are based on the colors of the hidden visuals but obfuscate any details
+        setting_use_pending_items: Hide timeline updates behind a click instead of automatically scrolling the feed
         username: Your username will be unique on %{domain}
         whole_word: When the keyword or phrase is alphanumeric only, it will only be applied if it matches the whole word
       featured_tag:
@@ -111,6 +112,7 @@ en:
         setting_theme: Site theme
         setting_unfollow_modal: Show confirmation dialog before unfollowing someone
         setting_use_blurhash: Show colorful gradients for hidden media
+        setting_use_pending_items: Slow mode
         severity: Severity
         type: Import type
         username: Username
diff --git a/config/settings.yml b/config/settings.yml
index ad71b6008..75cb2dc85 100644
--- a/config/settings.yml
+++ b/config/settings.yml
@@ -33,6 +33,7 @@ defaults: &defaults
   aggregate_reblogs: true
   advanced_layout: false
   use_blurhash: true
+  use_pending_items: false
   notification_emails:
     follow: false
     reblog: false
-- 
cgit 


From 91544a6cb54e54b7cbfb266553d29763d6b68176 Mon Sep 17 00:00:00 2001
From: ThibG <thib@sitedethib.com>
Date: Tue, 16 Jul 2019 14:25:56 +0200
Subject: Remove unused Account#magic_key (#11327)

---
 app/models/account.rb | 15 ---------------
 1 file changed, 15 deletions(-)

(limited to 'app/models')

diff --git a/app/models/account.rb b/app/models/account.rb
index a22b7fd7c..adf4586fa 100644
--- a/app/models/account.rb
+++ b/app/models/account.rb
@@ -301,21 +301,6 @@ class Account < ApplicationRecord
     self.fields = tmp
   end
 
-  def magic_key
-    modulus, exponent = [keypair.public_key.n, keypair.public_key.e].map do |component|
-      result = []
-
-      until component.zero?
-        result << [component % 256].pack('C')
-        component >>= 8
-      end
-
-      result.reverse.join
-    end
-
-    (['RSA'] + [modulus, exponent].map { |n| Base64.urlsafe_encode64(n) }).join('.')
-  end
-
   def subscription(webhook_url)
     @subscription ||= OStatus2::Subscription.new(remote_url, secret: secret, webhook: webhook_url, hub: hub_url)
   end
-- 
cgit 


From 729723f857d11434c0f78d63fe16537d77f1c77c Mon Sep 17 00:00:00 2001
From: Eugen Rochko <eugen@zeonfederated.com>
Date: Tue, 16 Jul 2019 06:30:47 +0200
Subject: Add option to disable real-time updates in web UI (#9984)

Fix #9031
Fix #7913
---
 app/controllers/settings/preferences_controller.rb |  1 +
 app/javascript/mastodon/actions/notifications.js   | 30 ++++++++++++---
 app/javascript/mastodon/actions/timelines.js       | 43 +++++++++++++++-------
 app/javascript/mastodon/compare_id.js              |  5 ++-
 app/javascript/mastodon/components/load_pending.js | 22 +++++++++++
 .../mastodon/components/scrollable_list.js         | 13 ++++++-
 .../components/column_settings.js                  |  2 +-
 .../notifications/components/setting_toggle.js     |  5 ++-
 .../mastodon/features/notifications/index.js       | 12 +++++-
 .../ui/containers/status_list_container.js         |  5 ++-
 app/javascript/mastodon/initial_state.js           |  1 +
 app/javascript/mastodon/reducers/notifications.js  | 30 ++++++++++-----
 app/javascript/mastodon/reducers/settings.js       |  6 ---
 app/javascript/mastodon/reducers/timelines.js      | 40 +++++++++++++-------
 app/lib/user_settings_decorator.rb                 |  5 +++
 app/models/user.rb                                 |  2 +-
 app/serializers/initial_state_serializer.rb        | 25 +++++++------
 .../settings/preferences/appearance/show.html.haml |  3 ++
 config/locales/simple_form.en.yml                  |  2 +
 config/settings.yml                                |  1 +
 20 files changed, 182 insertions(+), 71 deletions(-)
 create mode 100644 app/javascript/mastodon/components/load_pending.js

(limited to 'app/models')

diff --git a/app/controllers/settings/preferences_controller.rb b/app/controllers/settings/preferences_controller.rb
index 451742d41..372f253cb 100644
--- a/app/controllers/settings/preferences_controller.rb
+++ b/app/controllers/settings/preferences_controller.rb
@@ -53,6 +53,7 @@ class Settings::PreferencesController < Settings::BaseController
       :setting_advanced_layout,
       :setting_default_content_type,
       :setting_use_blurhash,
+      :setting_use_pending_items,
       notification_emails: %i(follow follow_request reblog favourite mention digest report pending_account),
       interactions: %i(must_be_follower must_be_following must_be_following_dm)
     )
diff --git a/app/javascript/mastodon/actions/notifications.js b/app/javascript/mastodon/actions/notifications.js
index 56c952cb0..d92d972bc 100644
--- a/app/javascript/mastodon/actions/notifications.js
+++ b/app/javascript/mastodon/actions/notifications.js
@@ -12,6 +12,8 @@ import { defineMessages } from 'react-intl';
 import { List as ImmutableList } from 'immutable';
 import { unescapeHTML } from '../utils/html';
 import { getFiltersRegex } from '../selectors';
+import { usePendingItems as preferPendingItems } from 'mastodon/initial_state';
+import compareId from 'mastodon/compare_id';
 
 export const NOTIFICATIONS_UPDATE      = 'NOTIFICATIONS_UPDATE';
 export const NOTIFICATIONS_UPDATE_NOOP = 'NOTIFICATIONS_UPDATE_NOOP';
@@ -22,8 +24,9 @@ export const NOTIFICATIONS_EXPAND_FAIL    = 'NOTIFICATIONS_EXPAND_FAIL';
 
 export const NOTIFICATIONS_FILTER_SET = 'NOTIFICATIONS_FILTER_SET';
 
-export const NOTIFICATIONS_CLEAR      = 'NOTIFICATIONS_CLEAR';
-export const NOTIFICATIONS_SCROLL_TOP = 'NOTIFICATIONS_SCROLL_TOP';
+export const NOTIFICATIONS_CLEAR        = 'NOTIFICATIONS_CLEAR';
+export const NOTIFICATIONS_SCROLL_TOP   = 'NOTIFICATIONS_SCROLL_TOP';
+export const NOTIFICATIONS_LOAD_PENDING = 'NOTIFICATIONS_LOAD_PENDING';
 
 defineMessages({
   mention: { id: 'notification.mention', defaultMessage: '{name} mentioned you' },
@@ -38,6 +41,10 @@ const fetchRelatedRelationships = (dispatch, notifications) => {
   }
 };
 
+export const loadPending = () => ({
+  type: NOTIFICATIONS_LOAD_PENDING,
+});
+
 export function updateNotifications(notification, intlMessages, intlLocale) {
   return (dispatch, getState) => {
     const showInColumn = getState().getIn(['settings', 'notifications', 'shows', notification.type], true);
@@ -69,6 +76,7 @@ export function updateNotifications(notification, intlMessages, intlLocale) {
       dispatch({
         type: NOTIFICATIONS_UPDATE,
         notification,
+        usePendingItems: preferPendingItems,
         meta: (playSound && !filtered) ? { sound: 'boop' } : undefined,
       });
 
@@ -122,10 +130,19 @@ export function expandNotifications({ maxId } = {}, done = noOp) {
         : excludeTypesFromFilter(activeFilter),
     };
 
-    if (!maxId && notifications.get('items').size > 0) {
-      params.since_id = notifications.getIn(['items', 0, 'id']);
+    if (!params.max_id && (notifications.get('items', ImmutableList()).size + notifications.get('pendingItems', ImmutableList()).size) > 0) {
+      const a = notifications.getIn(['pendingItems', 0, 'id']);
+      const b = notifications.getIn(['items', 0, 'id']);
+
+      if (a && b && compareId(a, b) > 0) {
+        params.since_id = a;
+      } else {
+        params.since_id = b || a;
+      }
     }
 
+    const isLoadingRecent = !!params.since_id;
+
     dispatch(expandNotificationsRequest(isLoadingMore));
 
     api(getState).get('/api/v1/notifications', { params }).then(response => {
@@ -134,7 +151,7 @@ export function expandNotifications({ maxId } = {}, done = noOp) {
       dispatch(importFetchedAccounts(response.data.map(item => item.account)));
       dispatch(importFetchedStatuses(response.data.map(item => item.status).filter(status => !!status)));
 
-      dispatch(expandNotificationsSuccess(response.data, next ? next.uri : null, isLoadingMore));
+      dispatch(expandNotificationsSuccess(response.data, next ? next.uri : null, isLoadingMore, isLoadingRecent && preferPendingItems));
       fetchRelatedRelationships(dispatch, response.data);
       done();
     }).catch(error => {
@@ -151,11 +168,12 @@ export function expandNotificationsRequest(isLoadingMore) {
   };
 };
 
-export function expandNotificationsSuccess(notifications, next, isLoadingMore) {
+export function expandNotificationsSuccess(notifications, next, isLoadingMore, usePendingItems) {
   return {
     type: NOTIFICATIONS_EXPAND_SUCCESS,
     notifications,
     next,
+    usePendingItems,
     skipLoading: !isLoadingMore,
   };
 };
diff --git a/app/javascript/mastodon/actions/timelines.js b/app/javascript/mastodon/actions/timelines.js
index 06c21b96b..7eeba2aa7 100644
--- a/app/javascript/mastodon/actions/timelines.js
+++ b/app/javascript/mastodon/actions/timelines.js
@@ -1,6 +1,8 @@
 import { importFetchedStatus, importFetchedStatuses } from './importer';
-import api, { getLinks } from '../api';
+import api, { getLinks } from 'mastodon/api';
 import { Map as ImmutableMap, List as ImmutableList } from 'immutable';
+import compareId from 'mastodon/compare_id';
+import { usePendingItems as preferPendingItems } from 'mastodon/initial_state';
 
 export const TIMELINE_UPDATE  = 'TIMELINE_UPDATE';
 export const TIMELINE_DELETE  = 'TIMELINE_DELETE';
@@ -10,10 +12,15 @@ export const TIMELINE_EXPAND_REQUEST = 'TIMELINE_EXPAND_REQUEST';
 export const TIMELINE_EXPAND_SUCCESS = 'TIMELINE_EXPAND_SUCCESS';
 export const TIMELINE_EXPAND_FAIL    = 'TIMELINE_EXPAND_FAIL';
 
-export const TIMELINE_SCROLL_TOP = 'TIMELINE_SCROLL_TOP';
+export const TIMELINE_SCROLL_TOP   = 'TIMELINE_SCROLL_TOP';
+export const TIMELINE_LOAD_PENDING = 'TIMELINE_LOAD_PENDING';
+export const TIMELINE_DISCONNECT   = 'TIMELINE_DISCONNECT';
+export const TIMELINE_CONNECT      = 'TIMELINE_CONNECT';
 
-export const TIMELINE_CONNECT    = 'TIMELINE_CONNECT';
-export const TIMELINE_DISCONNECT = 'TIMELINE_DISCONNECT';
+export const loadPending = timeline => ({
+  type: TIMELINE_LOAD_PENDING,
+  timeline,
+});
 
 export function updateTimeline(timeline, status, accept) {
   return dispatch => {
@@ -27,6 +34,7 @@ export function updateTimeline(timeline, status, accept) {
       type: TIMELINE_UPDATE,
       timeline,
       status,
+      usePendingItems: preferPendingItems,
     });
   };
 };
@@ -71,8 +79,15 @@ export function expandTimeline(timelineId, path, params = {}, done = noOp) {
       return;
     }
 
-    if (!params.max_id && !params.pinned && timeline.get('items', ImmutableList()).size > 0) {
-      params.since_id = timeline.getIn(['items', 0]);
+    if (!params.max_id && !params.pinned && (timeline.get('items', ImmutableList()).size + timeline.get('pendingItems', ImmutableList()).size) > 0) {
+      const a = timeline.getIn(['pendingItems', 0]);
+      const b = timeline.getIn(['items', 0]);
+
+      if (a && b && compareId(a, b) > 0) {
+        params.since_id = a;
+      } else {
+        params.since_id = b || a;
+      }
     }
 
     const isLoadingRecent = !!params.since_id;
@@ -82,7 +97,7 @@ export function expandTimeline(timelineId, path, params = {}, done = noOp) {
     api(getState).get(path, { params }).then(response => {
       const next = getLinks(response).refs.find(link => link.rel === 'next');
       dispatch(importFetchedStatuses(response.data));
-      dispatch(expandTimelineSuccess(timelineId, response.data, next ? next.uri : null, response.code === 206, isLoadingRecent, isLoadingMore));
+      dispatch(expandTimelineSuccess(timelineId, response.data, next ? next.uri : null, response.code === 206, isLoadingRecent, isLoadingMore, isLoadingRecent && preferPendingItems));
       done();
     }).catch(error => {
       dispatch(expandTimelineFail(timelineId, error, isLoadingMore));
@@ -115,7 +130,7 @@ export function expandTimelineRequest(timeline, isLoadingMore) {
   };
 };
 
-export function expandTimelineSuccess(timeline, statuses, next, partial, isLoadingRecent, isLoadingMore) {
+export function expandTimelineSuccess(timeline, statuses, next, partial, isLoadingRecent, isLoadingMore, usePendingItems) {
   return {
     type: TIMELINE_EXPAND_SUCCESS,
     timeline,
@@ -123,6 +138,7 @@ export function expandTimelineSuccess(timeline, statuses, next, partial, isLoadi
     next,
     partial,
     isLoadingRecent,
+    usePendingItems,
     skipLoading: !isLoadingMore,
   };
 };
@@ -151,9 +167,8 @@ export function connectTimeline(timeline) {
   };
 };
 
-export function disconnectTimeline(timeline) {
-  return {
-    type: TIMELINE_DISCONNECT,
-    timeline,
-  };
-};
+export const disconnectTimeline = timeline => ({
+  type: TIMELINE_DISCONNECT,
+  timeline,
+  usePendingItems: preferPendingItems,
+});
diff --git a/app/javascript/mastodon/compare_id.js b/app/javascript/mastodon/compare_id.js
index aaff66481..66cf51c4b 100644
--- a/app/javascript/mastodon/compare_id.js
+++ b/app/javascript/mastodon/compare_id.js
@@ -1,10 +1,11 @@
-export default function compareId(id1, id2) {
+export default function compareId (id1, id2) {
   if (id1 === id2) {
     return 0;
   }
+
   if (id1.length === id2.length) {
     return id1 > id2 ? 1 : -1;
   } else {
     return id1.length > id2.length ? 1 : -1;
   }
-}
+};
diff --git a/app/javascript/mastodon/components/load_pending.js b/app/javascript/mastodon/components/load_pending.js
new file mode 100644
index 000000000..7e2702403
--- /dev/null
+++ b/app/javascript/mastodon/components/load_pending.js
@@ -0,0 +1,22 @@
+import React from 'react';
+import { FormattedMessage } from 'react-intl';
+import PropTypes from 'prop-types';
+
+export default class LoadPending extends React.PureComponent {
+
+  static propTypes = {
+    onClick: PropTypes.func,
+    count: PropTypes.number,
+  }
+
+  render() {
+    const { count } = this.props;
+
+    return (
+      <button className='load-more load-gap' onClick={this.props.onClick}>
+        <FormattedMessage id='load_pending' defaultMessage='{count, plural, one {# new item} other {# new items}}' values={{ count }} />
+      </button>
+    );
+  }
+
+}
diff --git a/app/javascript/mastodon/components/scrollable_list.js b/app/javascript/mastodon/components/scrollable_list.js
index 0376cf85a..553c16352 100644
--- a/app/javascript/mastodon/components/scrollable_list.js
+++ b/app/javascript/mastodon/components/scrollable_list.js
@@ -3,6 +3,7 @@ import { ScrollContainer } from 'react-router-scroll-4';
 import PropTypes from 'prop-types';
 import IntersectionObserverArticleContainer from '../containers/intersection_observer_article_container';
 import LoadMore from './load_more';
+import LoadPending from './load_pending';
 import IntersectionObserverWrapper from '../features/ui/util/intersection_observer_wrapper';
 import { throttle } from 'lodash';
 import { List as ImmutableList } from 'immutable';
@@ -21,6 +22,7 @@ export default class ScrollableList extends PureComponent {
   static propTypes = {
     scrollKey: PropTypes.string.isRequired,
     onLoadMore: PropTypes.func,
+    onLoadPending: PropTypes.func,
     onScrollToTop: PropTypes.func,
     onScroll: PropTypes.func,
     trackScroll: PropTypes.bool,
@@ -28,6 +30,7 @@ export default class ScrollableList extends PureComponent {
     isLoading: PropTypes.bool,
     showLoading: PropTypes.bool,
     hasMore: PropTypes.bool,
+    numPending: PropTypes.number,
     prepend: PropTypes.node,
     alwaysPrepend: PropTypes.bool,
     emptyMessage: PropTypes.node,
@@ -225,12 +228,18 @@ export default class ScrollableList extends PureComponent {
     this.props.onLoadMore();
   }
 
+  handleLoadPending = e => {
+    e.preventDefault();
+    this.props.onLoadPending();
+  }
+
   render () {
-    const { children, scrollKey, trackScroll, shouldUpdateScroll, showLoading, isLoading, hasMore, prepend, alwaysPrepend, emptyMessage, onLoadMore } = this.props;
+    const { children, scrollKey, trackScroll, shouldUpdateScroll, showLoading, isLoading, hasMore, numPending, prepend, alwaysPrepend, emptyMessage, onLoadMore } = this.props;
     const { fullscreen } = this.state;
     const childrenCount = React.Children.count(children);
 
     const loadMore     = (hasMore && onLoadMore) ? <LoadMore visible={!isLoading} onClick={this.handleLoadMore} /> : null;
+    const loadPending  = (numPending > 0) ? <LoadPending count={numPending} onClick={this.handleLoadPending} /> : null;
     let scrollableArea = null;
 
     if (showLoading) {
@@ -251,6 +260,8 @@ export default class ScrollableList extends PureComponent {
           <div role='feed' className='item-list'>
             {prepend}
 
+            {loadPending}
+
             {React.Children.map(this.props.children, (child, index) => (
               <IntersectionObserverArticleContainer
                 key={child.key}
diff --git a/app/javascript/mastodon/features/community_timeline/components/column_settings.js b/app/javascript/mastodon/features/community_timeline/components/column_settings.js
index 8250190a7..0cb6db883 100644
--- a/app/javascript/mastodon/features/community_timeline/components/column_settings.js
+++ b/app/javascript/mastodon/features/community_timeline/components/column_settings.js
@@ -20,7 +20,7 @@ class ColumnSettings extends React.PureComponent {
     return (
       <div>
         <div className='column-settings__row'>
-          <SettingToggle settings={settings} settingPath={['other', 'onlyMedia']} onChange={onChange} label={<FormattedMessage id='community.column_settings.media_only' defaultMessage='Media Only' />} />
+          <SettingToggle settings={settings} settingPath={['other', 'onlyMedia']} onChange={onChange} label={<FormattedMessage id='community.column_settings.media_only' defaultMessage='Media only' />} />
         </div>
       </div>
     );
diff --git a/app/javascript/mastodon/features/notifications/components/setting_toggle.js b/app/javascript/mastodon/features/notifications/components/setting_toggle.js
index 7aec16d2e..e6f593ef8 100644
--- a/app/javascript/mastodon/features/notifications/components/setting_toggle.js
+++ b/app/javascript/mastodon/features/notifications/components/setting_toggle.js
@@ -11,6 +11,7 @@ export default class SettingToggle extends React.PureComponent {
     settingPath: PropTypes.array.isRequired,
     label: PropTypes.node.isRequired,
     onChange: PropTypes.func.isRequired,
+    defaultValue: PropTypes.bool,
   }
 
   onChange = ({ target }) => {
@@ -18,12 +19,12 @@ export default class SettingToggle extends React.PureComponent {
   }
 
   render () {
-    const { prefix, settings, settingPath, label } = this.props;
+    const { prefix, settings, settingPath, label, defaultValue } = this.props;
     const id = ['setting-toggle', prefix, ...settingPath].filter(Boolean).join('-');
 
     return (
       <div className='setting-toggle'>
-        <Toggle id={id} checked={settings.getIn(settingPath)} onChange={this.onChange} onKeyDown={this.onKeyDown} />
+        <Toggle id={id} checked={settings.getIn(settingPath, defaultValue)} onChange={this.onChange} onKeyDown={this.onKeyDown} />
         <label htmlFor={id} className='setting-toggle__label'>{label}</label>
       </div>
     );
diff --git a/app/javascript/mastodon/features/notifications/index.js b/app/javascript/mastodon/features/notifications/index.js
index 006c45657..df4ad6f2a 100644
--- a/app/javascript/mastodon/features/notifications/index.js
+++ b/app/javascript/mastodon/features/notifications/index.js
@@ -4,7 +4,7 @@ import PropTypes from 'prop-types';
 import ImmutablePropTypes from 'react-immutable-proptypes';
 import Column from '../../components/column';
 import ColumnHeader from '../../components/column_header';
-import { expandNotifications, scrollTopNotifications } from '../../actions/notifications';
+import { expandNotifications, scrollTopNotifications, loadPending } from '../../actions/notifications';
 import { addColumn, removeColumn, moveColumn } from '../../actions/columns';
 import NotificationContainer from './containers/notification_container';
 import { defineMessages, injectIntl, FormattedMessage } from 'react-intl';
@@ -41,6 +41,7 @@ const mapStateToProps = state => ({
   isLoading: state.getIn(['notifications', 'isLoading'], true),
   isUnread: state.getIn(['notifications', 'unread']) > 0,
   hasMore: state.getIn(['notifications', 'hasMore']),
+  numPending: state.getIn(['notifications', 'pendingItems'], ImmutableList()).size,
 });
 
 export default @connect(mapStateToProps)
@@ -58,6 +59,7 @@ class Notifications extends React.PureComponent {
     isUnread: PropTypes.bool,
     multiColumn: PropTypes.bool,
     hasMore: PropTypes.bool,
+    numPending: PropTypes.number,
   };
 
   static defaultProps = {
@@ -80,6 +82,10 @@ class Notifications extends React.PureComponent {
     this.props.dispatch(expandNotifications({ maxId: last && last.get('id') }));
   }, 300, { leading: true });
 
+  handleLoadPending = () => {
+    this.props.dispatch(loadPending());
+  };
+
   handleScrollToTop = debounce(() => {
     this.props.dispatch(scrollTopNotifications(true));
   }, 100);
@@ -136,7 +142,7 @@ class Notifications extends React.PureComponent {
   }
 
   render () {
-    const { intl, notifications, shouldUpdateScroll, isLoading, isUnread, columnId, multiColumn, hasMore, showFilterBar } = this.props;
+    const { intl, notifications, shouldUpdateScroll, isLoading, isUnread, columnId, multiColumn, hasMore, numPending, showFilterBar } = this.props;
     const pinned = !!columnId;
     const emptyMessage = <FormattedMessage id='empty_column.notifications' defaultMessage="You don't have any notifications yet. Interact with others to start the conversation." />;
 
@@ -178,8 +184,10 @@ class Notifications extends React.PureComponent {
         isLoading={isLoading}
         showLoading={isLoading && notifications.size === 0}
         hasMore={hasMore}
+        numPending={numPending}
         emptyMessage={emptyMessage}
         onLoadMore={this.handleLoadOlder}
+        onLoadPending={this.handleLoadPending}
         onScrollToTop={this.handleScrollToTop}
         onScroll={this.handleScroll}
         shouldUpdateScroll={shouldUpdateScroll}
diff --git a/app/javascript/mastodon/features/ui/containers/status_list_container.js b/app/javascript/mastodon/features/ui/containers/status_list_container.js
index 3df5b7bea..7b8eb652b 100644
--- a/app/javascript/mastodon/features/ui/containers/status_list_container.js
+++ b/app/javascript/mastodon/features/ui/containers/status_list_container.js
@@ -1,6 +1,6 @@
 import { connect } from 'react-redux';
 import StatusList from '../../../components/status_list';
-import { scrollTopTimeline } from '../../../actions/timelines';
+import { scrollTopTimeline, loadPending } from '../../../actions/timelines';
 import { Map as ImmutableMap, List as ImmutableList } from 'immutable';
 import { createSelector } from 'reselect';
 import { debounce } from 'lodash';
@@ -37,6 +37,7 @@ const makeMapStateToProps = () => {
     isLoading: state.getIn(['timelines', timelineId, 'isLoading'], true),
     isPartial: state.getIn(['timelines', timelineId, 'isPartial'], false),
     hasMore:   state.getIn(['timelines', timelineId, 'hasMore']),
+    numPending: state.getIn(['timelines', timelineId, 'pendingItems'], ImmutableList()).size,
   });
 
   return mapStateToProps;
@@ -52,6 +53,8 @@ const mapDispatchToProps = (dispatch, { timelineId }) => ({
     dispatch(scrollTopTimeline(timelineId, false));
   }, 100),
 
+  onLoadPending: () => dispatch(loadPending(timelineId)),
+
 });
 
 export default connect(makeMapStateToProps, mapDispatchToProps)(StatusList);
diff --git a/app/javascript/mastodon/initial_state.js b/app/javascript/mastodon/initial_state.js
index 7df2a90bc..3c3c80e99 100644
--- a/app/javascript/mastodon/initial_state.js
+++ b/app/javascript/mastodon/initial_state.js
@@ -22,5 +22,6 @@ export const profile_directory = getMeta('profile_directory');
 export const isStaff = getMeta('is_staff');
 export const forceSingleColumn = !getMeta('advanced_layout');
 export const useBlurhash = getMeta('use_blurhash');
+export const usePendingItems = getMeta('use_pending_items');
 
 export default initialState;
diff --git a/app/javascript/mastodon/reducers/notifications.js b/app/javascript/mastodon/reducers/notifications.js
index 4d9604de9..e94a4946b 100644
--- a/app/javascript/mastodon/reducers/notifications.js
+++ b/app/javascript/mastodon/reducers/notifications.js
@@ -6,6 +6,7 @@ import {
   NOTIFICATIONS_FILTER_SET,
   NOTIFICATIONS_CLEAR,
   NOTIFICATIONS_SCROLL_TOP,
+  NOTIFICATIONS_LOAD_PENDING,
 } from '../actions/notifications';
 import {
   ACCOUNT_BLOCK_SUCCESS,
@@ -16,6 +17,7 @@ import { Map as ImmutableMap, List as ImmutableList } from 'immutable';
 import compareId from '../compare_id';
 
 const initialState = ImmutableMap({
+  pendingItems: ImmutableList(),
   items: ImmutableList(),
   hasMore: true,
   top: false,
@@ -31,7 +33,11 @@ const notificationToMap = notification => ImmutableMap({
   status: notification.status ? notification.status.id : null,
 });
 
-const normalizeNotification = (state, notification) => {
+const normalizeNotification = (state, notification, usePendingItems) => {
+  if (usePendingItems) {
+    return state.update('pendingItems', list => list.unshift(notificationToMap(notification)));
+  }
+
   const top = state.get('top');
 
   if (!top) {
@@ -47,7 +53,7 @@ const normalizeNotification = (state, notification) => {
   });
 };
 
-const expandNormalizedNotifications = (state, notifications, next) => {
+const expandNormalizedNotifications = (state, notifications, next, usePendingItems) => {
   let items = ImmutableList();
 
   notifications.forEach((n, i) => {
@@ -56,7 +62,7 @@ const expandNormalizedNotifications = (state, notifications, next) => {
 
   return state.withMutations(mutable => {
     if (!items.isEmpty()) {
-      mutable.update('items', list => {
+      mutable.update(usePendingItems ? 'pendingItems' : 'items', list => {
         const lastIndex = 1 + list.findLastIndex(
           item => item !== null && (compareId(item.get('id'), items.last().get('id')) > 0 || item.get('id') === items.last().get('id'))
         );
@@ -78,7 +84,8 @@ const expandNormalizedNotifications = (state, notifications, next) => {
 };
 
 const filterNotifications = (state, relationship) => {
-  return state.update('items', list => list.filterNot(item => item !== null && item.get('account') === relationship.id));
+  const helper = list => list.filterNot(item => item !== null && item.get('account') === relationship.id);
+  return state.update('items', helper).update('pendingItems', helper);
 };
 
 const updateTop = (state, top) => {
@@ -90,34 +97,37 @@ const updateTop = (state, top) => {
 };
 
 const deleteByStatus = (state, statusId) => {
-  return state.update('items', list => list.filterNot(item => item !== null && item.get('status') === statusId));
+  const helper = list => list.filterNot(item => item !== null && item.get('status') === statusId);
+  return state.update('items', helper).update('pendingItems', helper);
 };
 
 export default function notifications(state = initialState, action) {
   switch(action.type) {
+  case NOTIFICATIONS_LOAD_PENDING:
+    return state.update('items', list => state.get('pendingItems').concat(list.take(40))).set('pendingItems', ImmutableList()).set('unread', 0);
   case NOTIFICATIONS_EXPAND_REQUEST:
     return state.set('isLoading', true);
   case NOTIFICATIONS_EXPAND_FAIL:
     return state.set('isLoading', false);
   case NOTIFICATIONS_FILTER_SET:
-    return state.set('items', ImmutableList()).set('hasMore', true);
+    return state.set('items', ImmutableList()).set('pendingItems', ImmutableList()).set('hasMore', true);
   case NOTIFICATIONS_SCROLL_TOP:
     return updateTop(state, action.top);
   case NOTIFICATIONS_UPDATE:
-    return normalizeNotification(state, action.notification);
+    return normalizeNotification(state, action.notification, action.usePendingItems);
   case NOTIFICATIONS_EXPAND_SUCCESS:
-    return expandNormalizedNotifications(state, action.notifications, action.next);
+    return expandNormalizedNotifications(state, action.notifications, action.next, action.usePendingItems);
   case ACCOUNT_BLOCK_SUCCESS:
     return filterNotifications(state, action.relationship);
   case ACCOUNT_MUTE_SUCCESS:
     return action.relationship.muting_notifications ? filterNotifications(state, action.relationship) : state;
   case NOTIFICATIONS_CLEAR:
-    return state.set('items', ImmutableList()).set('hasMore', false);
+    return state.set('items', ImmutableList()).set('pendingItems', ImmutableList()).set('hasMore', false);
   case TIMELINE_DELETE:
     return deleteByStatus(state, action.id);
   case TIMELINE_DISCONNECT:
     return action.timeline === 'home' ?
-      state.update('items', items => items.first() ? items.unshift(null) : items) :
+      state.update(action.usePendingItems ? 'pendingItems' : 'items', items => items.first() ? items.unshift(null) : items) :
       state;
   default:
     return state;
diff --git a/app/javascript/mastodon/reducers/settings.js b/app/javascript/mastodon/reducers/settings.js
index a0eea137f..033bfc999 100644
--- a/app/javascript/mastodon/reducers/settings.js
+++ b/app/javascript/mastodon/reducers/settings.js
@@ -10,8 +10,6 @@ import uuid from '../uuid';
 const initialState = ImmutableMap({
   saved: true,
 
-  onboarded: false,
-
   skinTone: 1,
 
   home: ImmutableMap({
@@ -74,10 +72,6 @@ const initialState = ImmutableMap({
       body: '',
     }),
   }),
-
-  trends: ImmutableMap({
-    show: true,
-  }),
 });
 
 const defaultColumns = fromJS([
diff --git a/app/javascript/mastodon/reducers/timelines.js b/app/javascript/mastodon/reducers/timelines.js
index 309a95a19..0b036f5fe 100644
--- a/app/javascript/mastodon/reducers/timelines.js
+++ b/app/javascript/mastodon/reducers/timelines.js
@@ -8,6 +8,7 @@ import {
   TIMELINE_SCROLL_TOP,
   TIMELINE_CONNECT,
   TIMELINE_DISCONNECT,
+  TIMELINE_LOAD_PENDING,
 } from '../actions/timelines';
 import {
   ACCOUNT_BLOCK_SUCCESS,
@@ -25,10 +26,11 @@ const initialTimeline = ImmutableMap({
   top: true,
   isLoading: false,
   hasMore: true,
+  pendingItems: ImmutableList(),
   items: ImmutableList(),
 });
 
-const expandNormalizedTimeline = (state, timeline, statuses, next, isPartial, isLoadingRecent) => {
+const expandNormalizedTimeline = (state, timeline, statuses, next, isPartial, isLoadingRecent, usePendingItems) => {
   return state.update(timeline, initialTimeline, map => map.withMutations(mMap => {
     mMap.set('isLoading', false);
     mMap.set('isPartial', isPartial);
@@ -38,7 +40,7 @@ const expandNormalizedTimeline = (state, timeline, statuses, next, isPartial, is
     if (timeline.endsWith(':pinned')) {
       mMap.set('items', statuses.map(status => status.get('id')));
     } else if (!statuses.isEmpty()) {
-      mMap.update('items', ImmutableList(), oldIds => {
+      mMap.update(usePendingItems ? 'pendingItems' : 'items', ImmutableList(), oldIds => {
         const newIds = statuses.map(status => status.get('id'));
 
         const lastIndex = oldIds.findLastIndex(id => id !== null && compareId(id, newIds.last()) >= 0) + 1;
@@ -57,7 +59,15 @@ const expandNormalizedTimeline = (state, timeline, statuses, next, isPartial, is
   }));
 };
 
-const updateTimeline = (state, timeline, status) => {
+const updateTimeline = (state, timeline, status, usePendingItems) => {
+  if (usePendingItems) {
+    if (state.getIn([timeline, 'pendingItems'], ImmutableList()).includes(status.get('id')) || state.getIn([timeline, 'items'], ImmutableList()).includes(status.get('id'))) {
+      return state;
+    }
+
+    return state.update(timeline, initialTimeline, map => map.update('pendingItems', list => list.unshift(status.get('id'))));
+  }
+
   const top        = state.getIn([timeline, 'top']);
   const ids        = state.getIn([timeline, 'items'], ImmutableList());
   const includesId = ids.includes(status.get('id'));
@@ -78,8 +88,10 @@ const updateTimeline = (state, timeline, status) => {
 
 const deleteStatus = (state, id, accountId, references, exclude_account = null) => {
   state.keySeq().forEach(timeline => {
-    if (exclude_account === null || (timeline !== `account:${exclude_account}` && !timeline.startsWith(`account:${exclude_account}:`)))
-      state = state.updateIn([timeline, 'items'], list => list.filterNot(item => item === id));
+    if (exclude_account === null || (timeline !== `account:${exclude_account}` && !timeline.startsWith(`account:${exclude_account}:`))) {
+      const helper = list => list.filterNot(item => item === id);
+      state = state.updateIn([timeline, 'items'], helper).updateIn([timeline, 'pendingItems'], helper);
+    }
   });
 
   // Remove reblogs of deleted status
@@ -109,11 +121,10 @@ const filterTimelines = (state, relationship, statuses) => {
   return state;
 };
 
-const filterTimeline = (timeline, state, relationship, statuses) =>
-  state.updateIn([timeline, 'items'], ImmutableList(), list =>
-    list.filterNot(statusId =>
-      statuses.getIn([statusId, 'account']) === relationship.id
-    ));
+const filterTimeline = (timeline, state, relationship, statuses) => {
+  const helper = list => list.filterNot(statusId => statuses.getIn([statusId, 'account']) === relationship.id);
+  return state.updateIn([timeline, 'items'], ImmutableList(), helper).updateIn([timeline, 'pendingItems'], ImmutableList(), helper);
+};
 
 const updateTop = (state, timeline, top) => {
   return state.update(timeline, initialTimeline, map => map.withMutations(mMap => {
@@ -124,14 +135,17 @@ const updateTop = (state, timeline, top) => {
 
 export default function timelines(state = initialState, action) {
   switch(action.type) {
+  case TIMELINE_LOAD_PENDING:
+    return state.update(action.timeline, initialTimeline, map =>
+      map.update('items', list => map.get('pendingItems').concat(list.take(40))).set('pendingItems', ImmutableList()).set('unread', 0));
   case TIMELINE_EXPAND_REQUEST:
     return state.update(action.timeline, initialTimeline, map => map.set('isLoading', true));
   case TIMELINE_EXPAND_FAIL:
     return state.update(action.timeline, initialTimeline, map => map.set('isLoading', false));
   case TIMELINE_EXPAND_SUCCESS:
-    return expandNormalizedTimeline(state, action.timeline, fromJS(action.statuses), action.next, action.partial, action.isLoadingRecent);
+    return expandNormalizedTimeline(state, action.timeline, fromJS(action.statuses), action.next, action.partial, action.isLoadingRecent, action.usePendingItems);
   case TIMELINE_UPDATE:
-    return updateTimeline(state, action.timeline, fromJS(action.status));
+    return updateTimeline(state, action.timeline, fromJS(action.status), action.usePendingItems);
   case TIMELINE_DELETE:
     return deleteStatus(state, action.id, action.accountId, action.references, action.reblogOf);
   case TIMELINE_CLEAR:
@@ -149,7 +163,7 @@ export default function timelines(state = initialState, action) {
     return state.update(
       action.timeline,
       initialTimeline,
-      map => map.set('online', false).update('items', items => items.first() ? items.unshift(null) : items)
+      map => map.set('online', false).update(action.usePendingItems ? 'pendingItems' : 'items', items => items.first() ? items.unshift(null) : items)
     );
   default:
     return state;
diff --git a/app/lib/user_settings_decorator.rb b/app/lib/user_settings_decorator.rb
index ac35fd005..51d8c0970 100644
--- a/app/lib/user_settings_decorator.rb
+++ b/app/lib/user_settings_decorator.rb
@@ -39,6 +39,7 @@ class UserSettingsDecorator
     user.settings['advanced_layout']     = advanced_layout_preference if change?('setting_advanced_layout')
     user.settings['default_content_type']= default_content_type_preference if change?('setting_default_content_type')
     user.settings['use_blurhash']        = use_blurhash_preference if change?('setting_use_blurhash')
+    user.settings['use_pending_items']   = use_pending_items_preference if change?('setting_use_pending_items')
   end
 
   def merged_notification_emails
@@ -137,6 +138,10 @@ class UserSettingsDecorator
     boolean_cast_setting 'setting_use_blurhash'
   end
 
+  def use_pending_items_preference
+    boolean_cast_setting 'setting_use_pending_items'
+  end
+
   def boolean_cast_setting(key)
     ActiveModel::Type::Boolean.new.cast(settings[key])
   end
diff --git a/app/models/user.rb b/app/models/user.rb
index 9bc3dd608..bb903a45c 100644
--- a/app/models/user.rb
+++ b/app/models/user.rb
@@ -106,7 +106,7 @@ class User < ApplicationRecord
   delegate :auto_play_gif, :default_sensitive, :unfollow_modal, :boost_modal, :favourite_modal, :delete_modal,
            :reduce_motion, :system_font_ui, :noindex, :flavour, :skin, :display_media, :hide_network, :hide_followers_count,
            :expand_spoilers, :default_language, :aggregate_reblogs, :show_application,
-           :advanced_layout, :default_content_type, :use_blurhash, to: :settings, prefix: :setting, allow_nil: false
+           :advanced_layout, :default_content_type, :use_blurhash, :use_pending_items, to: :settings, prefix: :setting, allow_nil: false
 
   attr_reader :invite_code
   attr_writer :external
diff --git a/app/serializers/initial_state_serializer.rb b/app/serializers/initial_state_serializer.rb
index e3e2775fb..e22059182 100644
--- a/app/serializers/initial_state_serializer.rb
+++ b/app/serializers/initial_state_serializer.rb
@@ -37,18 +37,19 @@ class InitialStateSerializer < ActiveModel::Serializer
     }
 
     if object.current_account
-      store[:me]              = object.current_account.id.to_s
-      store[:unfollow_modal]  = object.current_account.user.setting_unfollow_modal
-      store[:boost_modal]     = object.current_account.user.setting_boost_modal
-      store[:favourite_modal] = object.current_account.user.setting_favourite_modal
-      store[:delete_modal]    = object.current_account.user.setting_delete_modal
-      store[:auto_play_gif]   = object.current_account.user.setting_auto_play_gif
-      store[:display_media]   = object.current_account.user.setting_display_media
-      store[:expand_spoilers] = object.current_account.user.setting_expand_spoilers
-      store[:reduce_motion]   = object.current_account.user.setting_reduce_motion
-      store[:advanced_layout] = object.current_account.user.setting_advanced_layout
-      store[:use_blurhash]    = object.current_account.user.setting_use_blurhash
-      store[:is_staff]        = object.current_account.user.staff?
+      store[:me]                = object.current_account.id.to_s
+      store[:unfollow_modal]    = object.current_account.user.setting_unfollow_modal
+      store[:boost_modal]       = object.current_account.user.setting_boost_modal
+      store[:favourite_modal]   = object.current_account.user.setting_favourite_modal
+      store[:delete_modal]      = object.current_account.user.setting_delete_modal
+      store[:auto_play_gif]     = object.current_account.user.setting_auto_play_gif
+      store[:display_media]     = object.current_account.user.setting_display_media
+      store[:expand_spoilers]   = object.current_account.user.setting_expand_spoilers
+      store[:reduce_motion]     = object.current_account.user.setting_reduce_motion
+      store[:advanced_layout]   = object.current_account.user.setting_advanced_layout
+      store[:use_blurhash]      = object.current_account.user.setting_use_blurhash
+      store[:use_pending_items] = object.current_account.user.setting_use_pending_items
+      store[:is_staff]          = object.current_account.user.staff?
       store[:default_content_type] = object.current_account.user.setting_default_content_type
     end
 
diff --git a/app/views/settings/preferences/appearance/show.html.haml b/app/views/settings/preferences/appearance/show.html.haml
index 1709c9c84..447958253 100644
--- a/app/views/settings/preferences/appearance/show.html.haml
+++ b/app/views/settings/preferences/appearance/show.html.haml
@@ -14,6 +14,9 @@
 
   %h4= t 'appearance.animations_and_accessibility'
 
+  .fields-group
+    = f.input :setting_use_pending_items, as: :boolean, wrapper: :with_label
+
   .fields-group
     = f.input :setting_auto_play_gif, as: :boolean, wrapper: :with_label, recommended: true
     = f.input :setting_reduce_motion, as: :boolean, wrapper: :with_label
diff --git a/config/locales/simple_form.en.yml b/config/locales/simple_form.en.yml
index b35b9b6ec..3db07cb8b 100644
--- a/config/locales/simple_form.en.yml
+++ b/config/locales/simple_form.en.yml
@@ -40,6 +40,7 @@ en:
         setting_show_application: The application you use to toot will be displayed in the detailed view of your toots
         setting_skin: Reskins the selected Mastodon flavour
         setting_use_blurhash: Gradients are based on the colors of the hidden visuals but obfuscate any details
+        setting_use_pending_items: Hide timeline updates behind a click instead of automatically scrolling the feed
         username: Your username will be unique on %{domain}
         whole_word: When the keyword or phrase is alphanumeric only, it will only be applied if it matches the whole word
       featured_tag:
@@ -122,6 +123,7 @@ en:
         setting_system_font_ui: Use system's default font
         setting_unfollow_modal: Show confirmation dialog before unfollowing someone
         setting_use_blurhash: Show colorful gradients for hidden media
+        setting_use_pending_items: Slow mode
         severity: Severity
         type: Import type
         username: Username
diff --git a/config/settings.yml b/config/settings.yml
index 07e24f8e2..bc97f650b 100644
--- a/config/settings.yml
+++ b/config/settings.yml
@@ -37,6 +37,7 @@ defaults: &defaults
   aggregate_reblogs: true
   advanced_layout: false
   use_blurhash: true
+  use_pending_items: false
   notification_emails:
     follow: false
     reblog: false
-- 
cgit 


From 7e2b6da57f7689757a50fa261c480445b1846703 Mon Sep 17 00:00:00 2001
From: ThibG <thib@sitedethib.com>
Date: Wed, 17 Jul 2019 21:09:15 +0200
Subject: Add setting to disable the anti-spam (#11296)
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

* Add environment variable to disable the anti-spam

* Move antispam setting to admin settings

* Fix typo

* antispam → spam_check
---
 app/controllers/admin/dashboard_controller.rb | 1 +
 app/lib/spam_check.rb                         | 6 +++++-
 app/models/form/admin_settings.rb             | 2 ++
 app/views/admin/dashboard/index.html.haml     | 2 ++
 app/views/admin/settings/edit.html.haml       | 3 +++
 config/locales/en.yml                         | 4 ++++
 config/settings.yml                           | 1 +
 7 files changed, 18 insertions(+), 1 deletion(-)

(limited to 'app/models')

diff --git a/app/controllers/admin/dashboard_controller.rb b/app/controllers/admin/dashboard_controller.rb
index f23ed1508..e74e4755f 100644
--- a/app/controllers/admin/dashboard_controller.rb
+++ b/app/controllers/admin/dashboard_controller.rb
@@ -30,6 +30,7 @@ module Admin
       @trending_hashtags     = TrendingTags.get(7)
       @profile_directory     = Setting.profile_directory
       @timeline_preview      = Setting.timeline_preview
+      @spam_check_enabled    = Setting.spam_check_enabled
     end
 
     private
diff --git a/app/lib/spam_check.rb b/app/lib/spam_check.rb
index 923d48a02..0cf1b8790 100644
--- a/app/lib/spam_check.rb
+++ b/app/lib/spam_check.rb
@@ -14,7 +14,7 @@ class SpamCheck
   end
 
   def skip?
-    already_flagged? || trusted? || no_unsolicited_mentions? || solicited_reply?
+    disabled? || already_flagged? || trusted? || no_unsolicited_mentions? || solicited_reply?
   end
 
   def spam?
@@ -80,6 +80,10 @@ class SpamCheck
 
   private
 
+  def disabled?
+    !Setting.spam_check_enabled
+  end
+
   def remove_mentions(text)
     return text.gsub(Account::MENTION_RE, '') if @status.local?
 
diff --git a/app/models/form/admin_settings.rb b/app/models/form/admin_settings.rb
index 86a86ec66..2c03c88a8 100644
--- a/app/models/form/admin_settings.rb
+++ b/app/models/form/admin_settings.rb
@@ -28,6 +28,7 @@ class Form::AdminSettings
     thumbnail
     hero
     mascot
+    spam_check_enabled
   ).freeze
 
   BOOLEAN_KEYS = %i(
@@ -39,6 +40,7 @@ class Form::AdminSettings
     show_known_fediverse_at_about_page
     preview_sensitive_media
     profile_directory
+    spam_check_enabled
   ).freeze
 
   UPLOAD_KEYS = %i(
diff --git a/app/views/admin/dashboard/index.html.haml b/app/views/admin/dashboard/index.html.haml
index d448e3862..77cc1a2a0 100644
--- a/app/views/admin/dashboard/index.html.haml
+++ b/app/views/admin/dashboard/index.html.haml
@@ -51,6 +51,8 @@
           = feature_hint(link_to(t('admin.dashboard.feature_timeline_preview'), edit_admin_settings_path), @timeline_preview)
         %li
           = feature_hint(link_to(t('admin.dashboard.feature_relay'), admin_relays_path), @relay_enabled)
+        %li
+          = feature_hint(link_to(t('admin.dashboard.feature_spam_check'), edit_admin_settings_path), @spam_check_enabled)
 
   .dashboard__widgets__versions
     %div
diff --git a/app/views/admin/settings/edit.html.haml b/app/views/admin/settings/edit.html.haml
index a67e6a2c8..b3bf3849c 100644
--- a/app/views/admin/settings/edit.html.haml
+++ b/app/views/admin/settings/edit.html.haml
@@ -66,6 +66,9 @@
   .fields-group
     = f.input :profile_directory, as: :boolean, wrapper: :with_label, label: t('admin.settings.profile_directory.title'), hint: t('admin.settings.profile_directory.desc_html')
 
+  .fields-group
+    = f.input :spam_check_enabled, as: :boolean, wrapper: :with_label, label: t('admin.settings.spam_check_enabled.title'), hint: t('admin.settings.spam_check_enabled.desc_html')
+
   %hr.spacer/
 
   .fields-group
diff --git a/config/locales/en.yml b/config/locales/en.yml
index 89251ad40..4e252945f 100644
--- a/config/locales/en.yml
+++ b/config/locales/en.yml
@@ -250,6 +250,7 @@ en:
       feature_profile_directory: Profile directory
       feature_registrations: Registrations
       feature_relay: Federation relay
+      feature_spam_check: Anti-spam
       feature_timeline_preview: Timeline preview
       features: Features
       hidden_service: Federation with hidden services
@@ -449,6 +450,9 @@ en:
         desc_html: You can write your own privacy policy, terms of service or other legalese. You can use HTML tags
         title: Custom terms of service
       site_title: Server name
+      spam_check_enabled:
+        desc_html: Mastodon can auto-silence and auto-report accounts based on measures such as detecting accounts who send repeated unsolicited messages. There may be false positives.
+        title: Anti-spam
       thumbnail:
         desc_html: Used for previews via OpenGraph and API. 1200x630px recommended
         title: Server thumbnail
diff --git a/config/settings.yml b/config/settings.yml
index 75cb2dc85..ad2970bb7 100644
--- a/config/settings.yml
+++ b/config/settings.yml
@@ -61,6 +61,7 @@ defaults: &defaults
   activity_api_enabled: true
   peers_api_enabled: true
   show_known_fediverse_at_about_page: true
+  spam_check_enabled: true
 
 development:
   <<: *defaults
-- 
cgit 


From 4906cabc6b8d91136973ad159f6a02209ea8e166 Mon Sep 17 00:00:00 2001
From: Eugen Rochko <eugen@zeonfederated.com>
Date: Thu, 18 Jul 2019 03:02:30 +0200
Subject: Add aac, m4a, 3gp to allowed audio formats (#11342)

Fix #11186
---
 app/models/media_attachment.rb | 16 ++++++++--------
 1 file changed, 8 insertions(+), 8 deletions(-)

(limited to 'app/models')

diff --git a/app/models/media_attachment.rb b/app/models/media_attachment.rb
index 30d9a9851..05be302af 100644
--- a/app/models/media_attachment.rb
+++ b/app/models/media_attachment.rb
@@ -26,14 +26,14 @@ class MediaAttachment < ApplicationRecord
 
   enum type: [:image, :gifv, :video, :unknown, :audio]
 
-  IMAGE_FILE_EXTENSIONS = ['.jpg', '.jpeg', '.png', '.gif', '.webp'].freeze
-  VIDEO_FILE_EXTENSIONS = ['.webm', '.mp4', '.m4v', '.mov'].freeze
-  AUDIO_FILE_EXTENSIONS = ['.ogg', '.oga', '.mp3', '.wav', '.flac', '.opus'].freeze
-
-  IMAGE_MIME_TYPES             = ['image/jpeg', 'image/png', 'image/gif', 'image/webp'].freeze
-  VIDEO_MIME_TYPES             = ['video/webm', 'video/mp4', 'video/quicktime', 'video/ogg'].freeze
-  VIDEO_CONVERTIBLE_MIME_TYPES = ['video/webm', 'video/quicktime'].freeze
-  AUDIO_MIME_TYPES             = ['audio/wave', 'audio/wav', 'audio/x-wav', 'audio/x-pn-wave', 'audio/ogg', 'audio/mpeg', 'audio/mp3', 'audio/webm', 'audio/flac'].freeze
+  IMAGE_FILE_EXTENSIONS = %w(.jpg .jpeg .png .gif .webp).freeze
+  VIDEO_FILE_EXTENSIONS = %w(.webm .mp4 .m4v .mov).freeze
+  AUDIO_FILE_EXTENSIONS = %w(.ogg .oga .mp3 .wav .flac .opus .aac .m4a .3gp).freeze
+
+  IMAGE_MIME_TYPES             = %w(image/jpeg image/png image/gif image/webp).freeze
+  VIDEO_MIME_TYPES             = %w(video/webm video/mp4 video/quicktime video/ogg).freeze
+  VIDEO_CONVERTIBLE_MIME_TYPES = %w(video/webm video/quicktime).freeze
+  AUDIO_MIME_TYPES             = %w(audio/wave audio/wav audio/x-wav audio/x-pn-wave audio/ogg audio/mpeg audio/mp3 audio/webm audio/flac audio/aac audio/m4a audio/3gpp).freeze
 
   BLURHASH_OPTIONS = {
     x_comp: 4,
-- 
cgit 


From 84e988479e956bcf522072879da936b100d1d46b Mon Sep 17 00:00:00 2001
From: Eugen Rochko <eugen@zeonfederated.com>
Date: Thu, 18 Jul 2019 03:02:56 +0200
Subject: Fix only one middle dot being recognized in hashtags (#11345)

Fix #10934
---
 app/models/tag.rb       |  2 +-
 spec/models/tag_spec.rb | 38 +++++++++++++++++++++++++++++++++++++-
 2 files changed, 38 insertions(+), 2 deletions(-)

(limited to 'app/models')

diff --git a/app/models/tag.rb b/app/models/tag.rb
index 7db76d157..01bace2bb 100644
--- a/app/models/tag.rb
+++ b/app/models/tag.rb
@@ -17,7 +17,7 @@ class Tag < ApplicationRecord
   has_many :featured_tags, dependent: :destroy, inverse_of: :tag
   has_one :account_tag_stat, dependent: :destroy
 
-  HASHTAG_NAME_RE = '[[:word:]_]*[[:alpha:]_·][[:word:]_]*'
+  HASHTAG_NAME_RE = '[[:word:]_][[:word:]_]*[[:alpha:]_·]*[[:word:]_·]*[[:word:]_]'
   HASHTAG_RE = /(?:^|[^\/\)\w])#(#{HASHTAG_NAME_RE})/i
 
   validates :name, presence: true, uniqueness: true, format: { with: /\A#{HASHTAG_NAME_RE}\z/i }
diff --git a/spec/models/tag_spec.rb b/spec/models/tag_spec.rb
index 1ca50cc29..161862392 100644
--- a/spec/models/tag_spec.rb
+++ b/spec/models/tag_spec.rb
@@ -31,7 +31,43 @@ RSpec.describe Tag, type: :model do
     end
 
     it 'matches #ａｅｓｔｈｅｔｉｃ' do
-      expect(subject.match('this is #ａｅｓｔｈｅｔｉｃ')).to_not be_nil
+      expect(subject.match('this is #ａｅｓｔｈｅｔｉｃ').to_s).to eq ' #ａｅｓｔｈｅｔｉｃ'
+    end
+
+    it 'matches digits at the start' do
+      expect(subject.match('hello #3d').to_s).to eq ' #3d'
+    end
+
+    it 'matches digits in the middle' do
+      expect(subject.match('hello #l33ts35k').to_s).to eq ' #l33ts35k'
+    end
+
+    it 'matches digits at the end' do
+      expect(subject.match('hello #world2016').to_s).to eq ' #world2016'
+    end
+
+    it 'matches underscores at the beginning' do
+      expect(subject.match('hello #_test').to_s).to eq ' #_test'
+    end
+
+    it 'matches underscores at the end' do
+      expect(subject.match('hello #test_').to_s).to eq ' #test_'
+    end
+
+    it 'matches underscores in the middle' do
+      expect(subject.match('hello #one_two_three').to_s).to eq ' #one_two_three'
+    end
+
+    it 'matches middle dots' do
+      expect(subject.match('hello #one·two·three').to_s).to eq ' #one·two·three'
+    end
+
+    it 'does not match middle dots at the start' do
+      expect(subject.match('hello #·one·two·three')).to be_nil
+    end
+
+    it 'does not match middle dots at the end' do
+      expect(subject.match('hello #one·two·three·').to_s).to eq ' #one·two·three'
     end
   end
 
-- 
cgit 


From 730c4053d642024b9949d72c8a9f1873532c6212 Mon Sep 17 00:00:00 2001
From: ThibG <thib@sitedethib.com>
Date: Fri, 19 Jul 2019 01:44:42 +0200
Subject: Add ActivityPub actor representing the entire server (#11321)

* Add support for an instance actor

* Skip username validation for local Application accounts

* Add migration script to create instance actor

* Make Codeclimate happy

* Switch to id -99 for instance actor

* Remove unused `icon` and `image` attributes from instance actor

* Use if/elsif/else instead of return + ternary operator

* Add instance actor to fresh installs

* Use instance actor as instance representative

Use instance actor for forwarding reports, relay operations, and spam
auto-reporting.

* Seed database in test environment

* Fix single-user mode

* Fix tests

* Fix specs to accomodate for an extra `Account`

* Auto-reject follows on instance actor

Following an instance actor might make sense, but we are not handling that
right now, so auto-reject.

* Fix webfinger lookup and serialization for instance actor

* Rename instance actor

* Make it clear in the HTML view that the instance actor should not be blocked

* Raise cache time for instance actor as there's no dynamic content

* Re-use /about/more with a flash message for instance actor profile
---
 app/controllers/about_controller.rb                |  4 +-
 app/controllers/application_controller.rb          |  2 +-
 app/controllers/home_controller.rb                 |  2 +-
 app/controllers/instance_actors_controller.rb      | 20 ++++++++
 app/javascript/styles/mastodon/containers.scss     |  4 ++
 app/lib/activitypub/activity/follow.rb             |  2 +-
 app/lib/activitypub/tag_manager.rb                 |  5 +-
 app/lib/webfinger_resource.rb                      |  6 +++
 app/models/account.rb                              |  8 ++-
 app/models/concerns/account_finder_concern.rb      |  2 +-
 app/serializers/activitypub/actor_serializer.rb    | 14 ++++--
 app/serializers/webfinger_serializer.rb            | 25 +++++++---
 app/views/about/more.html.haml                     |  2 +
 app/views/well_known/webfinger/show.xml.ruby       | 57 ++++++++++++++--------
 config/locales/en.yml                              |  3 ++
 config/routes.rb                                   |  4 ++
 db/migrate/20190715164535_add_instance_actor.rb    |  9 ++++
 db/schema.rb                                       |  2 +-
 db/seeds.rb                                        |  4 +-
 spec/models/account_spec.rb                        | 12 ++---
 spec/services/fetch_remote_account_service_spec.rb |  1 -
 spec/services/fetch_resource_service_spec.rb       |  4 +-
 spec/spec_helper.rb                                |  1 +
 23 files changed, 141 insertions(+), 52 deletions(-)
 create mode 100644 app/controllers/instance_actors_controller.rb
 create mode 100644 db/migrate/20190715164535_add_instance_actor.rb

(limited to 'app/models')

diff --git a/app/controllers/about_controller.rb b/app/controllers/about_controller.rb
index 52fb1dc1b..33bac9bbc 100644
--- a/app/controllers/about_controller.rb
+++ b/app/controllers/about_controller.rb
@@ -11,7 +11,9 @@ class AboutController < ApplicationController
 
   def show; end
 
-  def more; end
+  def more
+    flash.now[:notice] = I18n.t('about.instance_actor_flash') if params[:instance_actor]
+  end
 
   def terms; end
 
diff --git a/app/controllers/application_controller.rb b/app/controllers/application_controller.rb
index 26f3b1def..51e9764d4 100644
--- a/app/controllers/application_controller.rb
+++ b/app/controllers/application_controller.rb
@@ -91,7 +91,7 @@ class ApplicationController < ActionController::Base
   end
 
   def single_user_mode?
-    @single_user_mode ||= Rails.configuration.x.single_user_mode && Account.exists?
+    @single_user_mode ||= Rails.configuration.x.single_user_mode && Account.where('id > 0').exists?
   end
 
   def use_seamless_external_login?
diff --git a/app/controllers/home_controller.rb b/app/controllers/home_controller.rb
index d1c525134..42493cd78 100644
--- a/app/controllers/home_controller.rb
+++ b/app/controllers/home_controller.rb
@@ -58,7 +58,7 @@ class HomeController < ApplicationController
     if request.path.start_with?('/web')
       new_user_session_path
     elsif single_user_mode?
-      short_account_path(Account.local.without_suspended.first)
+      short_account_path(Account.local.without_suspended.where('id > 0').first)
     else
       about_path
     end
diff --git a/app/controllers/instance_actors_controller.rb b/app/controllers/instance_actors_controller.rb
new file mode 100644
index 000000000..41f33602e
--- /dev/null
+++ b/app/controllers/instance_actors_controller.rb
@@ -0,0 +1,20 @@
+# frozen_string_literal: true
+
+class InstanceActorsController < ApplicationController
+  include AccountControllerConcern
+
+  def show
+    expires_in 10.minutes, public: true
+    render json: @account, content_type: 'application/activity+json', serializer: ActivityPub::ActorSerializer, adapter: ActivityPub::Adapter, fields: restrict_fields_to
+  end
+
+  private
+
+  def set_account
+    @account = Account.find(-99)
+  end
+
+  def restrict_fields_to
+    %i(id type preferred_username inbox public_key endpoints url manually_approves_followers)
+  end
+end
diff --git a/app/javascript/styles/mastodon/containers.scss b/app/javascript/styles/mastodon/containers.scss
index 3564bf07b..2b6794ee2 100644
--- a/app/javascript/styles/mastodon/containers.scss
+++ b/app/javascript/styles/mastodon/containers.scss
@@ -145,6 +145,10 @@
     min-height: 100%;
   }
 
+  .flash-message {
+    margin-bottom: 10px;
+  }
+
   @media screen and (max-width: 738px) {
     grid-template-columns: minmax(0, 50%) minmax(0, 50%);
 
diff --git a/app/lib/activitypub/activity/follow.rb b/app/lib/activitypub/activity/follow.rb
index 3eb88339a..28f1da19f 100644
--- a/app/lib/activitypub/activity/follow.rb
+++ b/app/lib/activitypub/activity/follow.rb
@@ -8,7 +8,7 @@ class ActivityPub::Activity::Follow < ActivityPub::Activity
 
     return if target_account.nil? || !target_account.local? || delete_arrived_first?(@json['id']) || @account.requested?(target_account)
 
-    if target_account.blocking?(@account) || target_account.domain_blocking?(@account.domain) || target_account.moved?
+    if target_account.blocking?(@account) || target_account.domain_blocking?(@account.domain) || target_account.moved? || target_account.instance_actor?
       reject_follow_request!(target_account)
       return
     end
diff --git a/app/lib/activitypub/tag_manager.rb b/app/lib/activitypub/tag_manager.rb
index 4d452f290..512272dbe 100644
--- a/app/lib/activitypub/tag_manager.rb
+++ b/app/lib/activitypub/tag_manager.rb
@@ -17,7 +17,7 @@ class ActivityPub::TagManager
 
     case target.object_type
     when :person
-      short_account_url(target)
+      target.instance_actor? ? about_more_url(instance_actor: true) : short_account_url(target)
     when :note, :comment, :activity
       return activity_account_status_url(target.account, target) if target.reblog?
       short_account_status_url(target.account, target)
@@ -29,7 +29,7 @@ class ActivityPub::TagManager
 
     case target.object_type
     when :person
-      account_url(target)
+      target.instance_actor? ? instance_actor_url : account_url(target)
     when :note, :comment, :activity
       return activity_account_status_url(target.account, target) if target.reblog?
       account_status_url(target.account, target)
@@ -119,6 +119,7 @@ class ActivityPub::TagManager
 
   def uri_to_local_id(uri, param = :id)
     path_params = Rails.application.routes.recognize_path(uri)
+    path_params[:username] = Rails.configuration.x.local_domain if path_params[:controller] == 'instance_actors'
     path_params[param]
   end
 
diff --git a/app/lib/webfinger_resource.rb b/app/lib/webfinger_resource.rb
index a54a702a2..22d78874a 100644
--- a/app/lib/webfinger_resource.rb
+++ b/app/lib/webfinger_resource.rb
@@ -23,11 +23,17 @@ class WebfingerResource
   def username_from_url
     if account_show_page?
       path_params[:username]
+    elsif instance_actor_page?
+      Rails.configuration.x.local_domain
     else
       raise ActiveRecord::RecordNotFound
     end
   end
 
+  def instance_actor_page?
+    path_params[:controller] == 'instance_actors'
+  end
+
   def account_show_page?
     path_params[:controller] == 'accounts' && path_params[:action] == 'show'
   end
diff --git a/app/models/account.rb b/app/models/account.rb
index adf4586fa..ccd116d6e 100644
--- a/app/models/account.rb
+++ b/app/models/account.rb
@@ -77,7 +77,7 @@ class Account < ApplicationRecord
   validates :username, format: { with: /\A#{USERNAME_RE}\z/i }, if: -> { !local? && will_save_change_to_username? }
 
   # Local user validations
-  validates :username, format: { with: /\A[a-z0-9_]+\z/i }, length: { maximum: 30 }, if: -> { local? && will_save_change_to_username? }
+  validates :username, format: { with: /\A[a-z0-9_]+\z/i }, length: { maximum: 30 }, if: -> { local? && will_save_change_to_username? && actor_type != 'Application' }
   validates_with UniqueUsernameValidator, if: -> { local? && will_save_change_to_username? }
   validates_with UnreservedUsernameValidator, if: -> { local? && will_save_change_to_username? }
   validates :display_name, length: { maximum: 30 }, if: -> { local? && will_save_change_to_display_name? }
@@ -139,6 +139,10 @@ class Account < ApplicationRecord
     %w(Application Service).include? actor_type
   end
 
+  def instance_actor?
+    id == -99
+  end
+
   alias bot bot?
 
   def bot=(val)
@@ -498,7 +502,7 @@ class Account < ApplicationRecord
   end
 
   def generate_keys
-    return unless local? && !Rails.env.test?
+    return unless local? && private_key.blank? && public_key.blank?
 
     keypair = OpenSSL::PKey::RSA.new(2048)
     self.private_key = keypair.to_pem
diff --git a/app/models/concerns/account_finder_concern.rb b/app/models/concerns/account_finder_concern.rb
index ccd7bfa12..a54c2174d 100644
--- a/app/models/concerns/account_finder_concern.rb
+++ b/app/models/concerns/account_finder_concern.rb
@@ -13,7 +13,7 @@ module AccountFinderConcern
     end
 
     def representative
-      find_local(Setting.site_contact_username.strip.gsub(/\A@/, '')) || Account.local.without_suspended.first
+      Account.find(-99)
     end
 
     def find_local(username)
diff --git a/app/serializers/activitypub/actor_serializer.rb b/app/serializers/activitypub/actor_serializer.rb
index 0644219fb..0bd7aed2e 100644
--- a/app/serializers/activitypub/actor_serializer.rb
+++ b/app/serializers/activitypub/actor_serializer.rb
@@ -39,11 +39,17 @@ class ActivityPub::ActorSerializer < ActivityPub::Serializer
   delegate :moved?, to: :object
 
   def id
-    account_url(object)
+    object.instance_actor? ? instance_actor_url : account_url(object)
   end
 
   def type
-    object.bot? ? 'Service' : 'Person'
+    if object.instance_actor?
+      'Application'
+    elsif object.bot?
+      'Service'
+    else
+      'Person'
+    end
   end
 
   def following
@@ -55,7 +61,7 @@ class ActivityPub::ActorSerializer < ActivityPub::Serializer
   end
 
   def inbox
-    account_inbox_url(object)
+    object.instance_actor? ? instance_actor_inbox_url : account_inbox_url(object)
   end
 
   def outbox
@@ -95,7 +101,7 @@ class ActivityPub::ActorSerializer < ActivityPub::Serializer
   end
 
   def url
-    short_account_url(object)
+    object.instance_actor? ? about_more_url(instance_actor: true) : short_account_url(object)
   end
 
   def avatar_exists?
diff --git a/app/serializers/webfinger_serializer.rb b/app/serializers/webfinger_serializer.rb
index f4af21551..008d0c182 100644
--- a/app/serializers/webfinger_serializer.rb
+++ b/app/serializers/webfinger_serializer.rb
@@ -10,15 +10,26 @@ class WebfingerSerializer < ActiveModel::Serializer
   end
 
   def aliases
-    [short_account_url(object), account_url(object)]
+    if object.instance_actor?
+      [instance_actor_url]
+    else
+      [short_account_url(object), account_url(object)]
+    end
   end
 
   def links
-    [
-      { rel: 'http://webfinger.net/rel/profile-page', type: 'text/html', href: short_account_url(object) },
-      { rel: 'http://schemas.google.com/g/2010#updates-from', type: 'application/atom+xml', href: account_url(object, format: 'atom') },
-      { rel: 'self', type: 'application/activity+json', href: account_url(object) },
-      { rel: 'http://ostatus.org/schema/1.0/subscribe', template: "#{authorize_interaction_url}?uri={uri}" },
-    ]
+    if object.instance_actor?
+      [
+        { rel: 'http://webfinger.net/rel/profile-page', type: 'text/html', href: about_more_url(instance_actor: true) },
+        { rel: 'self', type: 'application/activity+json', href: instance_actor_url },
+      ]
+    else
+      [
+        { rel: 'http://webfinger.net/rel/profile-page', type: 'text/html', href: short_account_url(object) },
+        { rel: 'http://schemas.google.com/g/2010#updates-from', type: 'application/atom+xml', href: account_url(object, format: 'atom') },
+        { rel: 'self', type: 'application/activity+json', href: account_url(object) },
+        { rel: 'http://ostatus.org/schema/1.0/subscribe', template: "#{authorize_interaction_url}?uri={uri}" },
+      ]
+    end
   end
 end
diff --git a/app/views/about/more.html.haml b/app/views/about/more.html.haml
index b248ed1d2..21431ef8e 100644
--- a/app/views/about/more.html.haml
+++ b/app/views/about/more.html.haml
@@ -43,5 +43,7 @@
           = mail_to @instance_presenter.site_contact_email, nil, title: @instance_presenter.site_contact_email
 
   .column-3
+    = render 'application/flashes'
+
     .box-widget
       .rich-formatting= @instance_presenter.site_extended_description.html_safe.presence || t('about.extended_description_html')
diff --git a/app/views/well_known/webfinger/show.xml.ruby b/app/views/well_known/webfinger/show.xml.ruby
index ae80df9d2..f5a54052a 100644
--- a/app/views/well_known/webfinger/show.xml.ruby
+++ b/app/views/well_known/webfinger/show.xml.ruby
@@ -4,30 +4,47 @@ doc << Ox::Element.new('XRD').tap do |xrd|
   xrd['xmlns'] = 'http://docs.oasis-open.org/ns/xri/xrd-1.0'
 
   xrd << (Ox::Element.new('Subject') << @account.to_webfinger_s)
-  xrd << (Ox::Element.new('Alias') << short_account_url(@account))
-  xrd << (Ox::Element.new('Alias') << account_url(@account))
 
-  xrd << Ox::Element.new('Link').tap do |link|
-    link['rel']      = 'http://webfinger.net/rel/profile-page'
-    link['type']     = 'text/html'
-    link['href']     = short_account_url(@account)
-  end
+  if @account.instance_actor?
+    xrd << (Ox::Element.new('Alias') << instance_actor_url)
 
-  xrd << Ox::Element.new('Link').tap do |link|
-    link['rel']      = 'http://schemas.google.com/g/2010#updates-from'
-    link['type']     = 'application/atom+xml'
-    link['href']     = account_url(@account, format: 'atom')
-  end
+    xrd << Ox::Element.new('Link').tap do |link|
+      link['rel']      = 'http://webfinger.net/rel/profile-page'
+      link['type']     = 'text/html'
+      link['href']     = about_more_url(instance_actor: true)
+    end
 
-  xrd << Ox::Element.new('Link').tap do |link|
-    link['rel']      = 'self'
-    link['type']     = 'application/activity+json'
-    link['href']     = account_url(@account)
-  end
+    xrd << Ox::Element.new('Link').tap do |link|
+      link['rel']      = 'self'
+      link['type']     = 'application/activity+json'
+      link['href']     = instance_actor_url
+    end
+  else
+    xrd << (Ox::Element.new('Alias') << short_account_url(@account))
+    xrd << (Ox::Element.new('Alias') << account_url(@account))
+
+    xrd << Ox::Element.new('Link').tap do |link|
+      link['rel']      = 'http://webfinger.net/rel/profile-page'
+      link['type']     = 'text/html'
+      link['href']     = short_account_url(@account)
+    end
+
+    xrd << Ox::Element.new('Link').tap do |link|
+      link['rel']      = 'http://schemas.google.com/g/2010#updates-from'
+      link['type']     = 'application/atom+xml'
+      link['href']     = account_url(@account, format: 'atom')
+    end
+
+    xrd << Ox::Element.new('Link').tap do |link|
+      link['rel']      = 'self'
+      link['type']     = 'application/activity+json'
+      link['href']     = account_url(@account)
+    end
 
-  xrd << Ox::Element.new('Link').tap do |link|
-    link['rel']      = 'http://ostatus.org/schema/1.0/subscribe'
-    link['template'] = "#{authorize_interaction_url}?acct={uri}"
+    xrd << Ox::Element.new('Link').tap do |link|
+      link['rel']      = 'http://ostatus.org/schema/1.0/subscribe'
+      link['template'] = "#{authorize_interaction_url}?acct={uri}"
+    end
   end
 end
 
diff --git a/config/locales/en.yml b/config/locales/en.yml
index 4e252945f..89c52b84a 100644
--- a/config/locales/en.yml
+++ b/config/locales/en.yml
@@ -24,6 +24,9 @@ en:
     generic_description: "%{domain} is one server in the network"
     get_apps: Try a mobile app
     hosted_on: Mastodon hosted on %{domain}
+    instance_actor_flash: |
+      This account is a virtual actor used to represent the server itself and not any individual user.
+      It is used for federation purposes and should not be blocked unless you want to block the whole instance, in which case you should use a domain block.
     learn_more: Learn more
     privacy_policy: Privacy policy
     see_whats_happening: See what's happening
diff --git a/config/routes.rb b/config/routes.rb
index 95f8a39ad..27b536641 100644
--- a/config/routes.rb
+++ b/config/routes.rb
@@ -28,6 +28,10 @@ Rails.application.routes.draw do
   get 'intent', to: 'intents#show'
   get 'custom.css', to: 'custom_css#show', as: :custom_css
 
+  resource :instance_actor, path: 'actor', only: [:show] do
+    resource :inbox, only: [:create], module: :activitypub
+  end
+
   devise_scope :user do
     get '/invite/:invite_code', to: 'auth/registrations#new', as: :public_invite
     match '/auth/finish_signup' => 'auth/confirmations#finish_signup', via: [:get, :patch], as: :finish_signup
diff --git a/db/migrate/20190715164535_add_instance_actor.rb b/db/migrate/20190715164535_add_instance_actor.rb
new file mode 100644
index 000000000..a26d54949
--- /dev/null
+++ b/db/migrate/20190715164535_add_instance_actor.rb
@@ -0,0 +1,9 @@
+class AddInstanceActor < ActiveRecord::Migration[5.2]
+  def up
+    Account.create!(id: -99, actor_type: 'Application', locked: true, username: Rails.configuration.x.local_domain)
+  end
+
+  def down
+    Account.find_by(id: -99, actor_type: 'Application').destroy!
+  end
+end
diff --git a/db/schema.rb b/db/schema.rb
index c7b6b9be6..a6a14827b 100644
--- a/db/schema.rb
+++ b/db/schema.rb
@@ -10,7 +10,7 @@
 #
 # It's strongly recommended that you check this file into your version control system.
 
-ActiveRecord::Schema.define(version: 2019_07_06_233204) do
+ActiveRecord::Schema.define(version: 2019_07_15_164535) do
 
   # These are extensions that must be enabled in order to support this database
   enable_extension "plpgsql"
diff --git a/db/seeds.rb b/db/seeds.rb
index 9a6e9dd78..5f43fbac8 100644
--- a/db/seeds.rb
+++ b/db/seeds.rb
@@ -1,7 +1,9 @@
 Doorkeeper::Application.create!(name: 'Web', superapp: true, redirect_uri: Doorkeeper.configuration.native_redirect_uri, scopes: 'read write follow')
 
+domain = ENV['LOCAL_DOMAIN'] || Rails.configuration.x.local_domain
+Account.create!(id: -99, actor_type: 'Application', locked: true, username: domain)
+
 if Rails.env.development?
-  domain = ENV['LOCAL_DOMAIN'] || Rails.configuration.x.local_domain
   admin  = Account.where(username: 'admin').first_or_initialize(username: 'admin')
   admin.save(validate: false)
   User.where(email: "admin@#{domain}").first_or_initialize(email: "admin@#{domain}", password: 'mastodonadmin', password_confirmation: 'mastodonadmin', confirmed_at: Time.now.utc, admin: true, account: admin, agreement: true, approved: true).save!
diff --git a/spec/models/account_spec.rb b/spec/models/account_spec.rb
index ce9ea250d..6495a6193 100644
--- a/spec/models/account_spec.rb
+++ b/spec/models/account_spec.rb
@@ -450,7 +450,7 @@ RSpec.describe Account, type: :model do
   describe '.domains' do
     it 'returns domains' do
       Fabricate(:account, domain: 'domain')
-      expect(Account.domains).to match_array(['domain'])
+      expect(Account.remote.domains).to match_array(['domain'])
     end
   end
 
@@ -665,7 +665,7 @@ RSpec.describe Account, type: :model do
           { username: 'b', domain: 'b' },
         ].map(&method(:Fabricate).curry(2).call(:account))
 
-        expect(Account.alphabetic).to eq matches
+        expect(Account.where('id > 0').alphabetic).to eq matches
       end
     end
 
@@ -732,7 +732,7 @@ RSpec.describe Account, type: :model do
         2.times { Fabricate(:account, domain: 'example.com') }
         Fabricate(:account, domain: 'example2.com')
 
-        results = Account.by_domain_accounts
+        results = Account.where('id > 0').by_domain_accounts
         expect(results.length).to eq 2
         expect(results.first.domain).to eq 'example.com'
         expect(results.first.accounts_count).to eq 2
@@ -745,7 +745,7 @@ RSpec.describe Account, type: :model do
       it 'returns an array of accounts who do not have a domain' do
         account_1 = Fabricate(:account, domain: nil)
         account_2 = Fabricate(:account, domain: 'example.com')
-        expect(Account.local).to match_array([account_1])
+        expect(Account.where('id > 0').local).to match_array([account_1])
       end
     end
 
@@ -756,14 +756,14 @@ RSpec.describe Account, type: :model do
           matches[index] = Fabricate(:account, domain: matches[index])
         end
 
-        expect(Account.partitioned).to match_array(matches)
+        expect(Account.where('id > 0').partitioned).to match_array(matches)
       end
     end
 
     describe 'recent' do
       it 'returns a relation of accounts sorted by recent creation' do
         matches = 2.times.map { Fabricate(:account) }
-        expect(Account.recent).to match_array(matches)
+        expect(Account.where('id > 0').recent).to match_array(matches)
       end
     end
 
diff --git a/spec/services/fetch_remote_account_service_spec.rb b/spec/services/fetch_remote_account_service_spec.rb
index b37445861..ee7325be2 100644
--- a/spec/services/fetch_remote_account_service_spec.rb
+++ b/spec/services/fetch_remote_account_service_spec.rb
@@ -4,7 +4,6 @@ RSpec.describe FetchRemoteAccountService, type: :service do
   let(:url) { 'https://example.com/alice' }
   let(:prefetched_body) { nil }
   let(:protocol) { :ostatus }
-  let!(:representative) { Fabricate(:account) }
 
   subject { FetchRemoteAccountService.new.call(url, prefetched_body, protocol) }
 
diff --git a/spec/services/fetch_resource_service_spec.rb b/spec/services/fetch_resource_service_spec.rb
index 98630966b..f836147d3 100644
--- a/spec/services/fetch_resource_service_spec.rb
+++ b/spec/services/fetch_resource_service_spec.rb
@@ -1,8 +1,6 @@
 require 'rails_helper'
 
 RSpec.describe FetchResourceService, type: :service do
-  let!(:representative) { Fabricate(:account) }
-
   describe '#call' do
     let(:url) { 'http://example.com' }
 
@@ -60,7 +58,7 @@ RSpec.describe FetchResourceService, type: :service do
 
       it 'signs request' do
         subject
-        expect(a_request(:get, url).with(headers: { 'Signature' => /keyId="#{Regexp.escape(ActivityPub::TagManager.instance.uri_for(representative) + '#main-key')}"/ })).to have_been_made
+        expect(a_request(:get, url).with(headers: { 'Signature' => /keyId="#{Regexp.escape(ActivityPub::TagManager.instance.uri_for(Account.representative) + '#main-key')}"/ })).to have_been_made
       end
 
       context 'when content type is application/atom+xml' do
diff --git a/spec/spec_helper.rb b/spec/spec_helper.rb
index 0cd1f91d0..45ba1bbd9 100644
--- a/spec/spec_helper.rb
+++ b/spec/spec_helper.rb
@@ -27,6 +27,7 @@ RSpec.configure do |config|
   end
 
   config.before :suite do
+    Rails.application.load_seed
     Chewy.strategy(:bypass)
   end
 
-- 
cgit 


From c37c1da41e1c490771a409ad1b12f3db55af4cee Mon Sep 17 00:00:00 2001
From: ThibG <thib@sitedethib.com>
Date: Fri, 19 Jul 2019 23:22:35 +0200
Subject: Disallow numeric-only hashtags (#11363)

* Add spec covering numeric-only hashtags

* Fix hashtag regex
---
 app/models/tag.rb       | 4 ++--
 spec/models/tag_spec.rb | 4 ++++
 2 files changed, 6 insertions(+), 2 deletions(-)

(limited to 'app/models')

diff --git a/app/models/tag.rb b/app/models/tag.rb
index 01bace2bb..b371d59c1 100644
--- a/app/models/tag.rb
+++ b/app/models/tag.rb
@@ -17,10 +17,10 @@ class Tag < ApplicationRecord
   has_many :featured_tags, dependent: :destroy, inverse_of: :tag
   has_one :account_tag_stat, dependent: :destroy
 
-  HASHTAG_NAME_RE = '[[:word:]_][[:word:]_]*[[:alpha:]_·]*[[:word:]_·]*[[:word:]_]'
+  HASHTAG_NAME_RE = '([[:word:]_][[:word:]_·]*[[:alpha:]_·][[:word:]_·]*[[:word:]_])|([[:word:]_]*[[:alpha:]][[:word:]_]*)'
   HASHTAG_RE = /(?:^|[^\/\)\w])#(#{HASHTAG_NAME_RE})/i
 
-  validates :name, presence: true, uniqueness: true, format: { with: /\A#{HASHTAG_NAME_RE}\z/i }
+  validates :name, presence: true, uniqueness: true, format: { with: /\A(#{HASHTAG_NAME_RE})\z/i }
 
   scope :discoverable, -> { joins(:account_tag_stat).where(AccountTagStat.arel_table[:accounts_count].gt(0)).where(account_tag_stats: { hidden: false }).order(Arel.sql('account_tag_stats.accounts_count desc')) }
   scope :hidden, -> { where(account_tag_stats: { hidden: true }) }
diff --git a/spec/models/tag_spec.rb b/spec/models/tag_spec.rb
index 161862392..9a30ceaa5 100644
--- a/spec/models/tag_spec.rb
+++ b/spec/models/tag_spec.rb
@@ -69,6 +69,10 @@ RSpec.describe Tag, type: :model do
     it 'does not match middle dots at the end' do
       expect(subject.match('hello #one·two·three·').to_s).to eq ' #one·two·three'
     end
+
+    it 'does not match purely-numeric hashtags' do
+      expect(subject.match('hello #0123456')).to be_nil
+    end
   end
 
   describe '#to_param' do
-- 
cgit 


From bd87e6667975bc3b5bfaf3e1cdff97e041ed4c98 Mon Sep 17 00:00:00 2001
From: Eugen Rochko <eugen@zeonfederated.com>
Date: Sun, 21 Jul 2019 04:08:00 +0200
Subject: Remove WebSub subscriptions (#11303)

---
 app/models/concerns/account_associations.rb        |  3 -
 app/models/subscription.rb                         | 62 --------------------
 app/policies/subscription_policy.rb                |  7 ---
 app/services/suspend_account_service.rb            |  1 -
 .../scheduler/subscriptions_cleanup_scheduler.rb   |  4 +-
 config/sidekiq.yml                                 |  3 -
 .../20190715031050_drop_subscriptions.rb           | 11 ++++
 db/schema.rb                                       | 14 -----
 spec/fabricators/subscription_fabricator.rb        |  7 ---
 spec/models/subscription_spec.rb                   | 67 ----------------------
 spec/policies/subscription_policy_spec.rb          | 24 --------
 .../services/batched_remove_status_service_spec.rb |  3 -
 spec/services/remove_status_service_spec.rb        |  3 -
 spec/services/suspend_account_service_spec.rb      |  8 +--
 14 files changed, 14 insertions(+), 203 deletions(-)
 delete mode 100644 app/models/subscription.rb
 delete mode 100644 app/policies/subscription_policy.rb
 create mode 100644 db/post_migrate/20190715031050_drop_subscriptions.rb
 delete mode 100644 spec/fabricators/subscription_fabricator.rb
 delete mode 100644 spec/models/subscription_spec.rb
 delete mode 100644 spec/policies/subscription_policy_spec.rb

(limited to 'app/models')

diff --git a/app/models/concerns/account_associations.rb b/app/models/concerns/account_associations.rb
index 0921e3252..1db7771c7 100644
--- a/app/models/concerns/account_associations.rb
+++ b/app/models/concerns/account_associations.rb
@@ -30,9 +30,6 @@ module AccountAssociations
     has_many :media_attachments, dependent: :destroy
     has_many :polls, dependent: :destroy
 
-    # PuSH subscriptions
-    has_many :subscriptions, dependent: :destroy
-
     # Report relationships
     has_many :reports, dependent: :destroy, inverse_of: :account
     has_many :targeted_reports, class_name: 'Report', foreign_key: :target_account_id, dependent: :destroy, inverse_of: :target_account
diff --git a/app/models/subscription.rb b/app/models/subscription.rb
deleted file mode 100644
index 79b81828d..000000000
--- a/app/models/subscription.rb
+++ /dev/null
@@ -1,62 +0,0 @@
-# frozen_string_literal: true
-# == Schema Information
-#
-# Table name: subscriptions
-#
-#  id                          :bigint(8)        not null, primary key
-#  callback_url                :string           default(""), not null
-#  secret                      :string
-#  expires_at                  :datetime
-#  confirmed                   :boolean          default(FALSE), not null
-#  created_at                  :datetime         not null
-#  updated_at                  :datetime         not null
-#  last_successful_delivery_at :datetime
-#  domain                      :string
-#  account_id                  :bigint(8)        not null
-#
-
-class Subscription < ApplicationRecord
-  MIN_EXPIRATION = 1.day.to_i
-  MAX_EXPIRATION = 30.days.to_i
-
-  belongs_to :account
-
-  validates :callback_url, presence: true
-  validates :callback_url, uniqueness: { scope: :account_id }
-
-  scope :confirmed, -> { where(confirmed: true) }
-  scope :future_expiration, -> { where(arel_table[:expires_at].gt(Time.now.utc)) }
-  scope :expired, -> { where(arel_table[:expires_at].lt(Time.now.utc)) }
-  scope :active, -> { confirmed.future_expiration }
-
-  def lease_seconds=(value)
-    self.expires_at = future_expiration(value)
-  end
-
-  def lease_seconds
-    (expires_at - Time.now.utc).to_i
-  end
-
-  def expired?
-    Time.now.utc > expires_at
-  end
-
-  before_validation :set_min_expiration
-
-  private
-
-  def future_expiration(value)
-    Time.now.utc + future_offset(value).seconds
-  end
-
-  def future_offset(seconds)
-    [
-      [MIN_EXPIRATION, seconds.to_i].max,
-      MAX_EXPIRATION,
-    ].min
-  end
-
-  def set_min_expiration
-    self.lease_seconds = 0 unless expires_at
-  end
-end
diff --git a/app/policies/subscription_policy.rb b/app/policies/subscription_policy.rb
deleted file mode 100644
index ac9a8a6c4..000000000
--- a/app/policies/subscription_policy.rb
+++ /dev/null
@@ -1,7 +0,0 @@
-# frozen_string_literal: true
-
-class SubscriptionPolicy < ApplicationPolicy
-  def index?
-    admin?
-  end
-end
diff --git a/app/services/suspend_account_service.rb b/app/services/suspend_account_service.rb
index 0ebe0b562..00cffcdfc 100644
--- a/app/services/suspend_account_service.rb
+++ b/app/services/suspend_account_service.rb
@@ -24,7 +24,6 @@ class SuspendAccountService < BaseService
     report_notes
     scheduled_statuses
     status_pins
-    subscriptions
   ).freeze
 
   ASSOCIATIONS_ON_DESTROY = %w(
diff --git a/app/workers/scheduler/subscriptions_cleanup_scheduler.rb b/app/workers/scheduler/subscriptions_cleanup_scheduler.rb
index 5fba120f6..75fe681a9 100644
--- a/app/workers/scheduler/subscriptions_cleanup_scheduler.rb
+++ b/app/workers/scheduler/subscriptions_cleanup_scheduler.rb
@@ -5,7 +5,5 @@ class Scheduler::SubscriptionsCleanupScheduler
 
   sidekiq_options unique: :until_executed, retry: 0
 
-  def perform
-    Subscription.expired.in_batches.delete_all
-  end
+  def perform; end
 end
diff --git a/config/sidekiq.yml b/config/sidekiq.yml
index 5c652792c..7f41b6607 100644
--- a/config/sidekiq.yml
+++ b/config/sidekiq.yml
@@ -21,9 +21,6 @@
   user_cleanup_scheduler:
     cron: '<%= Random.rand(0..59) %> <%= Random.rand(4..6) %> * * *'
     class: Scheduler::UserCleanupScheduler
-  subscriptions_cleanup_scheduler:
-    cron: '<%= Random.rand(0..59) %> <%= Random.rand(1..3) %> * * 0'
-    class: Scheduler::SubscriptionsCleanupScheduler
   ip_cleanup_scheduler:
     cron: '<%= Random.rand(0..59) %> <%= Random.rand(3..5) %> * * *'
     class: Scheduler::IpCleanupScheduler
diff --git a/db/post_migrate/20190715031050_drop_subscriptions.rb b/db/post_migrate/20190715031050_drop_subscriptions.rb
new file mode 100644
index 000000000..3719afe4a
--- /dev/null
+++ b/db/post_migrate/20190715031050_drop_subscriptions.rb
@@ -0,0 +1,11 @@
+# frozen_string_literal: true
+
+class DropSubscriptions < ActiveRecord::Migration[5.2]
+  def up
+    drop_table :subscriptions
+  end
+
+  def down
+    raise ActiveRecord::IrreversibleMigration
+  end
+end
diff --git a/db/schema.rb b/db/schema.rb
index a6a14827b..6319dd932 100644
--- a/db/schema.rb
+++ b/db/schema.rb
@@ -648,19 +648,6 @@ ActiveRecord::Schema.define(version: 2019_07_15_164535) do
     t.index ["tag_id", "status_id"], name: "index_statuses_tags_on_tag_id_and_status_id", unique: true
   end
 
-  create_table "subscriptions", force: :cascade do |t|
-    t.string "callback_url", default: "", null: false
-    t.string "secret"
-    t.datetime "expires_at"
-    t.boolean "confirmed", default: false, null: false
-    t.datetime "created_at", null: false
-    t.datetime "updated_at", null: false
-    t.datetime "last_successful_delivery_at"
-    t.string "domain"
-    t.bigint "account_id", null: false
-    t.index ["account_id", "callback_url"], name: "index_subscriptions_on_account_id_and_callback_url", unique: true
-  end
-
   create_table "tags", force: :cascade do |t|
     t.string "name", default: "", null: false
     t.datetime "created_at", null: false
@@ -821,7 +808,6 @@ ActiveRecord::Schema.define(version: 2019_07_15_164535) do
   add_foreign_key "statuses", "statuses", column: "reblog_of_id", on_delete: :cascade
   add_foreign_key "statuses_tags", "statuses", on_delete: :cascade
   add_foreign_key "statuses_tags", "tags", name: "fk_3081861e21", on_delete: :cascade
-  add_foreign_key "subscriptions", "accounts", name: "fk_9847d1cbb5", on_delete: :cascade
   add_foreign_key "tombstones", "accounts", on_delete: :cascade
   add_foreign_key "user_invite_requests", "users", on_delete: :cascade
   add_foreign_key "users", "accounts", name: "fk_50500f500d", on_delete: :cascade
diff --git a/spec/fabricators/subscription_fabricator.rb b/spec/fabricators/subscription_fabricator.rb
deleted file mode 100644
index 347dab5df..000000000
--- a/spec/fabricators/subscription_fabricator.rb
+++ /dev/null
@@ -1,7 +0,0 @@
-Fabricator(:subscription) do
-  account
-  callback_url "http://example.com/callback"
-  secret       "foobar"
-  expires_at   "2016-11-28 11:30:07"
-  confirmed    false
-end
diff --git a/spec/models/subscription_spec.rb b/spec/models/subscription_spec.rb
deleted file mode 100644
index b83979d13..000000000
--- a/spec/models/subscription_spec.rb
+++ /dev/null
@@ -1,67 +0,0 @@
-require 'rails_helper'
-
-RSpec.describe Subscription, type: :model do
-  let(:alice) { Fabricate(:account, username: 'alice') }
-
-  subject { Fabricate(:subscription, account: alice) }
-
-  describe '#expired?' do
-    it 'return true when expires_at is past' do
-      subject.expires_at = 2.days.ago
-      expect(subject.expired?).to be true
-    end
-
-    it 'return false when expires_at is future' do
-      subject.expires_at = 2.days.from_now
-      expect(subject.expired?).to be false
-    end
-  end
-
-  describe 'lease_seconds' do
-    it 'returns the time remaining until expiration' do
-      datetime = 1.day.from_now
-      subscription = Subscription.new(expires_at: datetime)
-      travel_to(datetime - 12.hours) do
-        expect(subscription.lease_seconds).to eq(12.hours)
-      end
-    end
-  end
-
-  describe 'lease_seconds=' do
-    it 'sets expires_at to min expiration when small value is provided' do
-      subscription = Subscription.new
-      datetime = 1.day.from_now
-      too_low = Subscription::MIN_EXPIRATION - 1000
-      travel_to(datetime) do
-        subscription.lease_seconds = too_low
-      end
-
-      expected = datetime + Subscription::MIN_EXPIRATION.seconds
-      expect(subscription.expires_at).to be_within(1.0).of(expected)
-    end
-
-    it 'sets expires_at to value when valid value is provided' do
-      subscription = Subscription.new
-      datetime = 1.day.from_now
-      valid = Subscription::MIN_EXPIRATION + 1000
-      travel_to(datetime) do
-        subscription.lease_seconds = valid
-      end
-
-      expected = datetime + valid.seconds
-      expect(subscription.expires_at).to be_within(1.0).of(expected)
-    end
-
-    it 'sets expires_at to max expiration when large value is provided' do
-      subscription = Subscription.new
-      datetime = 1.day.from_now
-      too_high = Subscription::MAX_EXPIRATION + 1000
-      travel_to(datetime) do
-        subscription.lease_seconds = too_high
-      end
-
-      expected = datetime + Subscription::MAX_EXPIRATION.seconds
-      expect(subscription.expires_at).to be_within(1.0).of(expected)
-    end
-  end
-end
diff --git a/spec/policies/subscription_policy_spec.rb b/spec/policies/subscription_policy_spec.rb
deleted file mode 100644
index 21d60c15f..000000000
--- a/spec/policies/subscription_policy_spec.rb
+++ /dev/null
@@ -1,24 +0,0 @@
-# frozen_string_literal: true
-
-require 'rails_helper'
-require 'pundit/rspec'
-
-RSpec.describe SubscriptionPolicy do
-  let(:subject) { described_class }
-  let(:admin)   { Fabricate(:user, admin: true).account }
-  let(:john)    { Fabricate(:user).account }
-
-  permissions :index? do
-    context 'admin?' do
-      it 'permits' do
-        expect(subject).to permit(admin, Subscription)
-      end
-    end
-
-    context '!admin?' do
-      it 'denies' do
-        expect(subject).to_not permit(john, Subscription)
-      end
-    end
-  end
-end
diff --git a/spec/services/batched_remove_status_service_spec.rb b/spec/services/batched_remove_status_service_spec.rb
index d52e7f484..f84256f18 100644
--- a/spec/services/batched_remove_status_service_spec.rb
+++ b/spec/services/batched_remove_status_service_spec.rb
@@ -14,11 +14,8 @@ RSpec.describe BatchedRemoveStatusService, type: :service do
   before do
     allow(Redis.current).to receive_messages(publish: nil)
 
-    stub_request(:post, 'http://example.com/push').to_return(status: 200, body: '', headers: {})
-    stub_request(:post, 'http://example.com/salmon').to_return(status: 200, body: '', headers: {})
     stub_request(:post, 'http://example.com/inbox').to_return(status: 200)
 
-    Fabricate(:subscription, account: alice, callback_url: 'http://example.com/push', confirmed: true, expires_at: 30.days.from_now)
     jeff.user.update(current_sign_in_at: Time.zone.now)
     jeff.follow!(alice)
     hank.follow!(alice)
diff --git a/spec/services/remove_status_service_spec.rb b/spec/services/remove_status_service_spec.rb
index 48191d47c..06676ec45 100644
--- a/spec/services/remove_status_service_spec.rb
+++ b/spec/services/remove_status_service_spec.rb
@@ -10,12 +10,9 @@ RSpec.describe RemoveStatusService, type: :service do
   let!(:bill)   { Fabricate(:account, username: 'bill', protocol: :activitypub, domain: 'example2.com', inbox_url: 'http://example2.com/inbox') }
 
   before do
-    stub_request(:post, 'http://example.com/push').to_return(status: 200, body: '', headers: {})
-    stub_request(:post, 'http://example.com/salmon').to_return(status: 200, body: '', headers: {})
     stub_request(:post, 'http://example.com/inbox').to_return(status: 200)
     stub_request(:post, 'http://example2.com/inbox').to_return(status: 200)
 
-    Fabricate(:subscription, account: alice, callback_url: 'http://example.com/push', confirmed: true, expires_at: 30.days.from_now)
     jeff.follow!(alice)
     hank.follow!(alice)
 
diff --git a/spec/services/suspend_account_service_spec.rb b/spec/services/suspend_account_service_spec.rb
index 896ac17a3..eebbbc12a 100644
--- a/spec/services/suspend_account_service_spec.rb
+++ b/spec/services/suspend_account_service_spec.rb
@@ -18,7 +18,6 @@ RSpec.describe SuspendAccountService, type: :service do
     let!(:favourite) { Fabricate(:favourite, account: account) }
     let!(:active_relationship) { Fabricate(:follow, account: account) }
     let!(:passive_relationship) { Fabricate(:follow, target_account: account) }
-    let!(:subscription) { Fabricate(:subscription, account: account) }
     let!(:remote_alice) { Fabricate(:account, inbox_url: 'https://alice.com/inbox', protocol: :activitypub) }
     let!(:remote_bob) { Fabricate(:account, inbox_url: 'https://bob.com/inbox', protocol: :activitypub) }
 
@@ -31,9 +30,8 @@ RSpec.describe SuspendAccountService, type: :service do
           account.favourites,
           account.active_relationships,
           account.passive_relationships,
-          account.subscriptions
         ].map(&:count)
-      }.from([1, 1, 1, 1, 1, 1, 1]).to([0, 0, 0, 0, 0, 0, 0])
+      }.from([1, 1, 1, 1, 1, 1]).to([0, 0, 0, 0, 0, 0])
     end
 
     it 'sends a delete actor activity to all known inboxes' do
@@ -62,7 +60,6 @@ RSpec.describe SuspendAccountService, type: :service do
     let!(:favourite) { Fabricate(:favourite, account: remote_bob) }
     let!(:active_relationship) { Fabricate(:follow, account: remote_bob, target_account: account) }
     let!(:passive_relationship) { Fabricate(:follow, target_account: remote_bob) }
-    let!(:subscription) { Fabricate(:subscription, account: remote_bob) }
 
     it 'deletes associated records' do
       is_expected.to change {
@@ -73,9 +70,8 @@ RSpec.describe SuspendAccountService, type: :service do
           remote_bob.favourites,
           remote_bob.active_relationships,
           remote_bob.passive_relationships,
-          remote_bob.subscriptions
         ].map(&:count)
-      }.from([1, 1, 1, 1, 1, 1, 1]).to([0, 0, 0, 0, 0, 0, 0])
+      }.from([1, 1, 1, 1, 1, 1]).to([0, 0, 0, 0, 0, 0])
     end
 
     it 'sends a reject follow to follwer inboxes' do
-- 
cgit 


From 964ae8eee593687f922c873fa7b378bb6e3e39bb Mon Sep 17 00:00:00 2001
From: Eugen Rochko <eugen@zeonfederated.com>
Date: Mon, 22 Jul 2019 10:48:50 +0200
Subject: Change unconfirmed user login behaviour (#11375)

Allow access to account settings, 2FA, authorized applications, and
account deletions to unconfirmed and pending users, as well as
users who had their accounts disabled. Suspended users cannot update
their e-mail or password or delete their account.

Display account status on account settings page, for example, when
an account is frozen, limited, unconfirmed or pending review.

After sign up, login users straight away and show a simple page that
tells them the status of their account with links to account settings
and logout, to reduce onboarding friction and allow users to correct
wrongly typed e-mail addresses.

Move the final sign-up step of SSO integrations to be the same
as above to reduce code duplication.
---
 app/controllers/about_controller.rb                |  2 +-
 app/controllers/api/base_controller.rb             |  2 +-
 app/controllers/application_controller.rb          |  6 +--
 app/controllers/auth/confirmations_controller.rb   | 21 +-------
 .../auth/omniauth_callbacks_controller.rb          |  2 +-
 app/controllers/auth/registrations_controller.rb   |  9 +++-
 app/controllers/auth/sessions_controller.rb        |  4 +-
 app/controllers/auth/setup_controller.rb           | 58 ++++++++++++++++++++++
 .../oauth/authorized_applications_controller.rb    |  2 +
 app/controllers/settings/deletes_controller.rb     |  7 +++
 app/controllers/settings/sessions_controller.rb    |  2 +
 .../confirmations_controller.rb                    |  2 +
 .../recovery_codes_controller.rb                   |  2 +
 .../two_factor_authentications_controller.rb       |  2 +
 app/javascript/styles/mastodon/admin.scss          | 58 +++++++++++++---------
 app/javascript/styles/mastodon/forms.scss          |  7 +++
 app/models/concerns/omniauthable.rb                |  2 +-
 app/models/user.rb                                 |  6 ++-
 .../auth/confirmations/finish_signup.html.haml     | 15 ------
 app/views/auth/registrations/_sessions.html.haml   |  4 +-
 app/views/auth/registrations/_status.html.haml     | 16 ++++++
 app/views/auth/registrations/edit.html.haml        | 35 +++++++------
 app/views/auth/setup/show.html.haml                | 23 +++++++++
 .../oauth/authorized_applications/index.html.haml  |  2 +-
 config/locales/en.yml                              |  9 +++-
 config/routes.rb                                   |  5 +-
 db/seeds.rb                                        |  2 +-
 spec/controllers/api/base_controller_spec.rb       | 42 +++++++++++++++-
 spec/controllers/application_controller_spec.rb    |  4 +-
 .../auth/confirmations_controller_spec.rb          | 41 ---------------
 .../auth/registrations_controller_spec.rb          | 25 +++++++---
 spec/controllers/auth/sessions_controller_spec.rb  |  4 +-
 .../settings/deletes_controller_spec.rb            | 17 +++++++
 spec/features/log_in_spec.rb                       |  4 +-
 spec/models/user_spec.rb                           |  4 +-
 35 files changed, 298 insertions(+), 148 deletions(-)
 create mode 100644 app/controllers/auth/setup_controller.rb
 delete mode 100644 app/views/auth/confirmations/finish_signup.html.haml
 create mode 100644 app/views/auth/registrations/_status.html.haml
 create mode 100644 app/views/auth/setup/show.html.haml

(limited to 'app/models')

diff --git a/app/controllers/about_controller.rb b/app/controllers/about_controller.rb
index 33bac9bbc..31cf17710 100644
--- a/app/controllers/about_controller.rb
+++ b/app/controllers/about_controller.rb
@@ -7,7 +7,7 @@ class AboutController < ApplicationController
   before_action :set_instance_presenter
   before_action :set_expires_in
 
-  skip_before_action :check_user_permissions, only: [:more, :terms]
+  skip_before_action :require_functional!, only: [:more, :terms]
 
   def show; end
 
diff --git a/app/controllers/api/base_controller.rb b/app/controllers/api/base_controller.rb
index eca558f42..6f33a1ea9 100644
--- a/app/controllers/api/base_controller.rb
+++ b/app/controllers/api/base_controller.rb
@@ -7,7 +7,7 @@ class Api::BaseController < ApplicationController
   include RateLimitHeaders
 
   skip_before_action :store_current_location
-  skip_before_action :check_user_permissions
+  skip_before_action :require_functional!
 
   before_action :set_cache_headers
 
diff --git a/app/controllers/application_controller.rb b/app/controllers/application_controller.rb
index b8a1faf77..41ce1a0ca 100644
--- a/app/controllers/application_controller.rb
+++ b/app/controllers/application_controller.rb
@@ -25,7 +25,7 @@ class ApplicationController < ActionController::Base
   rescue_from Mastodon::NotPermittedError, with: :forbidden
 
   before_action :store_current_location, except: :raise_not_found, unless: :devise_controller?
-  before_action :check_user_permissions, if: :user_signed_in?
+  before_action :require_functional!, if: :user_signed_in?
 
   def raise_not_found
     raise ActionController::RoutingError, "No route matches #{params[:unmatched_route]}"
@@ -57,8 +57,8 @@ class ApplicationController < ActionController::Base
     forbidden unless current_user&.staff?
   end
 
-  def check_user_permissions
-    forbidden if current_user.disabled? || current_user.account.suspended?
+  def require_functional!
+    redirect_to edit_user_registration_path unless current_user.functional?
   end
 
   def after_sign_out_path_for(_resource_or_scope)
diff --git a/app/controllers/auth/confirmations_controller.rb b/app/controllers/auth/confirmations_controller.rb
index c28c7471c..0d7c6e7c2 100644
--- a/app/controllers/auth/confirmations_controller.rb
+++ b/app/controllers/auth/confirmations_controller.rb
@@ -4,34 +4,15 @@ class Auth::ConfirmationsController < Devise::ConfirmationsController
   layout 'auth'
 
   before_action :set_body_classes
-  before_action :set_user, only: [:finish_signup]
 
-  def finish_signup
-    return unless request.patch? && params[:user]
-
-    if @user.update(user_params)
-      @user.skip_reconfirmation!
-      bypass_sign_in(@user)
-      redirect_to root_path, notice: I18n.t('devise.confirmations.send_instructions')
-    else
-      @show_errors = true
-    end
-  end
+  skip_before_action :require_functional!
 
   private
 
-  def set_user
-    @user = current_user
-  end
-
   def set_body_classes
     @body_classes = 'lighter'
   end
 
-  def user_params
-    params.require(:user).permit(:email)
-  end
-
   def after_confirmation_path_for(_resource_name, user)
     if user.created_by_application && truthy_param?(:redirect_to_app)
       user.created_by_application.redirect_uri
diff --git a/app/controllers/auth/omniauth_callbacks_controller.rb b/app/controllers/auth/omniauth_callbacks_controller.rb
index bbf63bed3..682c77016 100644
--- a/app/controllers/auth/omniauth_callbacks_controller.rb
+++ b/app/controllers/auth/omniauth_callbacks_controller.rb
@@ -27,7 +27,7 @@ class Auth::OmniauthCallbacksController < Devise::OmniauthCallbacksController
     if resource.email_verified?
       root_path
     else
-      finish_signup_path
+      auth_setup_path(missing_email: '1')
     end
   end
 end
diff --git a/app/controllers/auth/registrations_controller.rb b/app/controllers/auth/registrations_controller.rb
index 83797cf1f..019caf9c1 100644
--- a/app/controllers/auth/registrations_controller.rb
+++ b/app/controllers/auth/registrations_controller.rb
@@ -9,6 +9,9 @@ class Auth::RegistrationsController < Devise::RegistrationsController
   before_action :set_sessions, only: [:edit, :update]
   before_action :set_instance_presenter, only: [:new, :create, :update]
   before_action :set_body_classes, only: [:new, :create, :edit, :update]
+  before_action :require_not_suspended!, only: [:update]
+
+  skip_before_action :require_functional!, only: [:edit, :update]
 
   def new
     super(&:build_invite_request)
@@ -43,7 +46,7 @@ class Auth::RegistrationsController < Devise::RegistrationsController
   end
 
   def after_sign_up_path_for(_resource)
-    new_user_session_path
+    auth_setup_path
   end
 
   def after_sign_in_path_for(_resource)
@@ -102,4 +105,8 @@ class Auth::RegistrationsController < Devise::RegistrationsController
   def set_sessions
     @sessions = current_user.session_activations
   end
+
+  def require_not_suspended!
+    forbidden if current_account.suspended?
+  end
 end
diff --git a/app/controllers/auth/sessions_controller.rb b/app/controllers/auth/sessions_controller.rb
index fb8615c31..7e6dbf19e 100644
--- a/app/controllers/auth/sessions_controller.rb
+++ b/app/controllers/auth/sessions_controller.rb
@@ -6,8 +6,10 @@ class Auth::SessionsController < Devise::SessionsController
   layout 'auth'
 
   skip_before_action :require_no_authentication, only: [:create]
-  skip_before_action :check_user_permissions, only: [:destroy]
+  skip_before_action :require_functional!
+
   prepend_before_action :authenticate_with_two_factor, if: :two_factor_enabled?, only: [:create]
+
   before_action :set_instance_presenter, only: [:new]
   before_action :set_body_classes
 
diff --git a/app/controllers/auth/setup_controller.rb b/app/controllers/auth/setup_controller.rb
new file mode 100644
index 000000000..46c5f2958
--- /dev/null
+++ b/app/controllers/auth/setup_controller.rb
@@ -0,0 +1,58 @@
+# frozen_string_literal: true
+
+class Auth::SetupController < ApplicationController
+  layout 'auth'
+
+  before_action :authenticate_user!
+  before_action :require_unconfirmed_or_pending!
+  before_action :set_body_classes
+  before_action :set_user
+
+  skip_before_action :require_functional!
+
+  def show
+    flash.now[:notice] = begin
+      if @user.pending?
+        I18n.t('devise.registrations.signed_up_but_pending')
+      else
+        I18n.t('devise.registrations.signed_up_but_unconfirmed')
+      end
+    end
+  end
+
+  def update
+    # This allows updating the e-mail without entering a password as is required
+    # on the account settings page; however, we only allow this for accounts
+    # that were not confirmed yet
+
+    if @user.update(user_params)
+      redirect_to auth_setup_path, notice: I18n.t('devise.confirmations.send_instructions')
+    else
+      render :show
+    end
+  end
+
+  helper_method :missing_email?
+
+  private
+
+  def require_unconfirmed_or_pending!
+    redirect_to root_path if current_user.confirmed? && current_user.approved?
+  end
+
+  def set_user
+    @user = current_user
+  end
+
+  def set_body_classes
+    @body_classes = 'lighter'
+  end
+
+  def user_params
+    params.require(:user).permit(:email)
+  end
+
+  def missing_email?
+    truthy_param?(:missing_email)
+  end
+end
diff --git a/app/controllers/oauth/authorized_applications_controller.rb b/app/controllers/oauth/authorized_applications_controller.rb
index f3d235366..fb8389034 100644
--- a/app/controllers/oauth/authorized_applications_controller.rb
+++ b/app/controllers/oauth/authorized_applications_controller.rb
@@ -7,6 +7,8 @@ class Oauth::AuthorizedApplicationsController < Doorkeeper::AuthorizedApplicatio
   before_action :authenticate_resource_owner!
   before_action :set_body_classes
 
+  skip_before_action :require_functional!
+
   include Localized
 
   def destroy
diff --git a/app/controllers/settings/deletes_controller.rb b/app/controllers/settings/deletes_controller.rb
index dd19aadf6..97fe4d328 100644
--- a/app/controllers/settings/deletes_controller.rb
+++ b/app/controllers/settings/deletes_controller.rb
@@ -5,6 +5,9 @@ class Settings::DeletesController < Settings::BaseController
 
   before_action :check_enabled_deletion
   before_action :authenticate_user!
+  before_action :require_not_suspended!
+
+  skip_before_action :require_functional!
 
   def show
     @confirmation = Form::DeleteConfirmation.new
@@ -29,4 +32,8 @@ class Settings::DeletesController < Settings::BaseController
   def delete_params
     params.require(:form_delete_confirmation).permit(:password)
   end
+
+  def require_not_suspended!
+    forbidden if current_account.suspended?
+  end
 end
diff --git a/app/controllers/settings/sessions_controller.rb b/app/controllers/settings/sessions_controller.rb
index 84ebb21f2..df5ace803 100644
--- a/app/controllers/settings/sessions_controller.rb
+++ b/app/controllers/settings/sessions_controller.rb
@@ -4,6 +4,8 @@ class Settings::SessionsController < Settings::BaseController
   before_action :authenticate_user!
   before_action :set_session, only: :destroy
 
+  skip_before_action :require_functional!
+
   def destroy
     @session.destroy!
     flash[:notice] = I18n.t('sessions.revoke_success')
diff --git a/app/controllers/settings/two_factor_authentication/confirmations_controller.rb b/app/controllers/settings/two_factor_authentication/confirmations_controller.rb
index 02652a36c..3145e092d 100644
--- a/app/controllers/settings/two_factor_authentication/confirmations_controller.rb
+++ b/app/controllers/settings/two_factor_authentication/confirmations_controller.rb
@@ -8,6 +8,8 @@ module Settings
       before_action :authenticate_user!
       before_action :ensure_otp_secret
 
+      skip_before_action :require_functional!
+
       def new
         prepare_two_factor_form
       end
diff --git a/app/controllers/settings/two_factor_authentication/recovery_codes_controller.rb b/app/controllers/settings/two_factor_authentication/recovery_codes_controller.rb
index 874bf532b..09a759860 100644
--- a/app/controllers/settings/two_factor_authentication/recovery_codes_controller.rb
+++ b/app/controllers/settings/two_factor_authentication/recovery_codes_controller.rb
@@ -7,6 +7,8 @@ module Settings
 
       before_action :authenticate_user!
 
+      skip_before_action :require_functional!
+
       def create
         @recovery_codes = current_user.generate_otp_backup_codes!
         current_user.save!
diff --git a/app/controllers/settings/two_factor_authentications_controller.rb b/app/controllers/settings/two_factor_authentications_controller.rb
index e12c43074..6904076e4 100644
--- a/app/controllers/settings/two_factor_authentications_controller.rb
+++ b/app/controllers/settings/two_factor_authentications_controller.rb
@@ -7,6 +7,8 @@ module Settings
     before_action :authenticate_user!
     before_action :verify_otp_required, only: [:create]
 
+    skip_before_action :require_functional!
+
     def show
       @confirmation = Form::TwoFactorConfirmation.new
     end
diff --git a/app/javascript/styles/mastodon/admin.scss b/app/javascript/styles/mastodon/admin.scss
index 373a10260..f625bc139 100644
--- a/app/javascript/styles/mastodon/admin.scss
+++ b/app/javascript/styles/mastodon/admin.scss
@@ -204,29 +204,6 @@ $content-width: 840px;
         border: 0;
       }
     }
-
-    .muted-hint {
-      color: $darker-text-color;
-
-      a {
-        color: $highlight-text-color;
-      }
-    }
-
-    .positive-hint {
-      color: $valid-value-color;
-      font-weight: 500;
-    }
-
-    .negative-hint {
-      color: $error-value-color;
-      font-weight: 500;
-    }
-
-    .neutral-hint {
-      color: $dark-text-color;
-      font-weight: 500;
-    }
   }
 
   @media screen and (max-width: $no-columns-breakpoint) {
@@ -249,6 +226,41 @@ $content-width: 840px;
   }
 }
 
+hr.spacer {
+  width: 100%;
+  border: 0;
+  margin: 20px 0;
+  height: 1px;
+}
+
+.muted-hint {
+  color: $darker-text-color;
+
+  a {
+    color: $highlight-text-color;
+  }
+}
+
+.positive-hint {
+  color: $valid-value-color;
+  font-weight: 500;
+}
+
+.negative-hint {
+  color: $error-value-color;
+  font-weight: 500;
+}
+
+.neutral-hint {
+  color: $dark-text-color;
+  font-weight: 500;
+}
+
+.warning-hint {
+  color: $gold-star;
+  font-weight: 500;
+}
+
 .filters {
   display: flex;
   flex-wrap: wrap;
diff --git a/app/javascript/styles/mastodon/forms.scss b/app/javascript/styles/mastodon/forms.scss
index 456ee4e0d..ac99124ea 100644
--- a/app/javascript/styles/mastodon/forms.scss
+++ b/app/javascript/styles/mastodon/forms.scss
@@ -300,6 +300,13 @@ code {
     }
   }
 
+  .input.static .label_input__wrapper {
+    font-size: 16px;
+    padding: 10px;
+    border: 1px solid $dark-text-color;
+    border-radius: 4px;
+  }
+
   input[type=text],
   input[type=number],
   input[type=email],
diff --git a/app/models/concerns/omniauthable.rb b/app/models/concerns/omniauthable.rb
index 283033083..b9c124841 100644
--- a/app/models/concerns/omniauthable.rb
+++ b/app/models/concerns/omniauthable.rb
@@ -43,7 +43,7 @@ module Omniauthable
       # Check if the user exists with provided email if the provider gives us a
       # verified email.  If no verified email was provided or the user already
       # exists, we assign a temporary email and ask the user to verify it on
-      # the next step via Auth::ConfirmationsController.finish_signup
+      # the next step via Auth::SetupController.show
 
       user = User.new(user_params_from_auth(auth))
       user.account.avatar_remote_url = auth.info.image if auth.info.image =~ /\A#{URI.regexp(%w(http https))}\z/
diff --git a/app/models/user.rb b/app/models/user.rb
index 31c99630c..474c77293 100644
--- a/app/models/user.rb
+++ b/app/models/user.rb
@@ -161,7 +161,11 @@ class User < ApplicationRecord
   end
 
   def active_for_authentication?
-    super && approved?
+    true
+  end
+
+  def functional?
+    confirmed? && approved? && !disabled? && !account.suspended?
   end
 
   def inactive_message
diff --git a/app/views/auth/confirmations/finish_signup.html.haml b/app/views/auth/confirmations/finish_signup.html.haml
deleted file mode 100644
index 9d09b74e1..000000000
--- a/app/views/auth/confirmations/finish_signup.html.haml
+++ /dev/null
@@ -1,15 +0,0 @@
-- content_for :page_title do
-  = t('auth.confirm_email')
-
-= simple_form_for(current_user, as: 'user', url: finish_signup_path, html: { role: 'form'}) do |f|
-  - if @show_errors && current_user.errors.any?
-    #error_explanation
-      - current_user.errors.full_messages.each do |msg|
-        = msg
-        %br
-
-  .fields-group
-    = f.input :email, wrapper: :with_label, required: true, hint: false
-
-  .actions
-    = f.submit t('auth.confirm_email'), class: 'button'
diff --git a/app/views/auth/registrations/_sessions.html.haml b/app/views/auth/registrations/_sessions.html.haml
index d7d96a1bb..395e36a9f 100644
--- a/app/views/auth/registrations/_sessions.html.haml
+++ b/app/views/auth/registrations/_sessions.html.haml
@@ -1,6 +1,8 @@
-%h4= t 'sessions.title'
+%h3= t 'sessions.title'
 %p.muted-hint= t 'sessions.explanation'
 
+%hr.spacer/
+
 .table-wrapper
   %table.table.inline-table
     %thead
diff --git a/app/views/auth/registrations/_status.html.haml b/app/views/auth/registrations/_status.html.haml
new file mode 100644
index 000000000..b38a83d67
--- /dev/null
+++ b/app/views/auth/registrations/_status.html.haml
@@ -0,0 +1,16 @@
+%h3= t('auth.status.account_status')
+
+- if @user.account.suspended?
+  %span.negative-hint= t('user_mailer.warning.explanation.suspend')
+- elsif @user.disabled?
+  %span.negative-hint= t('user_mailer.warning.explanation.disable')
+- elsif @user.account.silenced?
+  %span.warning-hint= t('user_mailer.warning.explanation.silence')
+- elsif !@user.confirmed?
+  %span.warning-hint= t('auth.status.confirming')
+- elsif !@user.approved?
+  %span.warning-hint= t('auth.status.pending')
+- else
+  %span.positive-hint= t('auth.status.functional')
+
+%hr.spacer/
diff --git a/app/views/auth/registrations/edit.html.haml b/app/views/auth/registrations/edit.html.haml
index 694461fdf..710ee5c68 100644
--- a/app/views/auth/registrations/edit.html.haml
+++ b/app/views/auth/registrations/edit.html.haml
@@ -1,25 +1,28 @@
 - content_for :page_title do
-  = t('auth.security')
+  = t('settings.account_settings')
+
+= render 'status'
+
+%h3= t('auth.security')
 
 = simple_form_for(resource, as: resource_name, url: registration_path(resource_name), html: { method: :put, class: 'auth_edit' }) do |f|
   = render 'shared/error_messages', object: resource
 
   - if !use_seamless_external_login? || resource.encrypted_password.present?
-    .fields-group
-      = f.input :email, wrapper: :with_label, input_html: { 'aria-label' => t('simple_form.labels.defaults.email') }, required: true, hint: false
-
-    .fields-group
-      = f.input :current_password, wrapper: :with_label, input_html: { 'aria-label' => t('simple_form.labels.defaults.current_password'), :autocomplete => 'off' }, required: true
-
-    .fields-group
-      = f.input :password, wrapper: :with_label, label: t('simple_form.labels.defaults.new_password'), input_html: { 'aria-label' => t('simple_form.labels.defaults.new_password'), :autocomplete => 'off' }, hint: false
-
-    .fields-group
-      = f.input :password_confirmation, wrapper: :with_label, label: t('simple_form.labels.defaults.confirm_new_password'), input_html: { 'aria-label' => t('simple_form.labels.defaults.confirm_new_password'), :autocomplete => 'off' }
-
+    .fields-row
+      .fields-row__column.fields-group.fields-row__column-6
+        = f.input :email, wrapper: :with_label, input_html: { 'aria-label' => t('simple_form.labels.defaults.email') }, required: true, disabled: current_account.suspended?
+      .fields-row__column.fields-group.fields-row__column-6
+        = f.input :current_password, wrapper: :with_label, input_html: { 'aria-label' => t('simple_form.labels.defaults.current_password'), :autocomplete => 'off' }, required: true, disabled: current_account.suspended?
+
+    .fields-row
+      .fields-row__column.fields-group.fields-row__column-6
+        = f.input :password, wrapper: :with_label, label: t('simple_form.labels.defaults.new_password'), input_html: { 'aria-label' => t('simple_form.labels.defaults.new_password'), :autocomplete => 'off' }, hint: t('simple_form.hints.defaults.password'), disabled: current_account.suspended?
+      .fields-row__column.fields-group.fields-row__column-6
+        = f.input :password_confirmation, wrapper: :with_label, label: t('simple_form.labels.defaults.confirm_new_password'), input_html: { 'aria-label' => t('simple_form.labels.defaults.confirm_new_password'), :autocomplete => 'off' }, disabled: current_account.suspended?
 
     .actions
-      = f.button :button, t('generic.save_changes'), type: :submit
+      = f.button :button, t('generic.save_changes'), type: :submit, class: 'button', disabled: current_account.suspended?
   - else
     %p.hint= t('users.seamless_external_login')
 
@@ -27,7 +30,7 @@
 
 = render 'sessions'
 
-- if open_deletion?
+- if open_deletion? && !current_account.suspended?
   %hr.spacer/
-  %h4= t('auth.delete_account')
+  %h3= t('auth.delete_account')
   %p.muted-hint= t('auth.delete_account_html', path: settings_delete_path)
diff --git a/app/views/auth/setup/show.html.haml b/app/views/auth/setup/show.html.haml
new file mode 100644
index 000000000..8bb44ca7f
--- /dev/null
+++ b/app/views/auth/setup/show.html.haml
@@ -0,0 +1,23 @@
+- content_for :page_title do
+  = t('auth.setup.title')
+
+- if missing_email?
+  = simple_form_for(@user, url: auth_setup_path) do |f|
+    = render 'shared/error_messages', object: @user
+
+    .fields-group
+      %p.hint= t('auth.setup.email_below_hint_html')
+
+    .fields-group
+      = f.input :email, required: true, hint: false, input_html: { 'aria-label' => t('simple_form.labels.defaults.email'), :autocomplete => 'off' }
+
+    .actions
+      = f.submit t('admin.accounts.change_email.label'), class: 'button'
+- else
+  .simple_form
+    %p.hint= t('auth.setup.email_settings_hint_html', email: content_tag(:strong, @user.email))
+
+.form-footer
+  %ul.no-list
+    %li= link_to t('settings.account_settings'), edit_user_registration_path
+    %li= link_to t('auth.logout'), destroy_user_session_path, data: { method: :delete }
diff --git a/app/views/oauth/authorized_applications/index.html.haml b/app/views/oauth/authorized_applications/index.html.haml
index 19af5f55d..7203d758d 100644
--- a/app/views/oauth/authorized_applications/index.html.haml
+++ b/app/views/oauth/authorized_applications/index.html.haml
@@ -17,7 +17,7 @@
               = application.name
             - else
               = link_to application.name, application.website, target: '_blank', rel: 'noopener'
-          %th!= application.scopes.map { |scope| t(scope, scope: [:doorkeeper, :scopes]) }.join('<br />')
+          %th!= application.scopes.map { |scope| t(scope, scope: [:doorkeeper, :scopes]) }.join(', ')
           %td= l application.created_at
           %td
             - unless application.superapp?
diff --git a/config/locales/en.yml b/config/locales/en.yml
index 89c52b84a..9e1be87be 100644
--- a/config/locales/en.yml
+++ b/config/locales/en.yml
@@ -524,7 +524,6 @@ en:
     apply_for_account: Request an invite
     change_password: Password
     checkbox_agreement_html: I agree to the <a href="%{rules_path}" target="_blank">server rules</a> and <a href="%{terms_path}" target="_blank">terms of service</a>
-    confirm_email: Confirm email
     delete_account: Delete account
     delete_account_html: If you wish to delete your account, you can <a href="%{path}">proceed here</a>. You will be asked for confirmation.
     didnt_get_confirmation: Didn't receive confirmation instructions?
@@ -544,6 +543,14 @@ en:
     reset_password: Reset password
     security: Security
     set_new_password: Set new password
+    setup:
+      email_below_hint_html: If the below e-mail address is incorrect, you can change it here and receive a new confirmation e-mail.
+      email_settings_hint_html: The confirmation e-mail was sent to %{email}. If that e-mail address is not correct, you can change it in account settings.
+      title: Setup
+    status:
+      account_status: Account status
+      confirming: Waiting for e-mail confirmation to be completed.
+      pending: Your application is pending review by our staff. This may take some time. You will receive an e-mail if your application is approved.
     trouble_logging_in: Trouble logging in?
   authorize_follow:
     already_following: You are already following this account
diff --git a/config/routes.rb b/config/routes.rb
index 27b536641..b6c215888 100644
--- a/config/routes.rb
+++ b/config/routes.rb
@@ -34,7 +34,10 @@ Rails.application.routes.draw do
 
   devise_scope :user do
     get '/invite/:invite_code', to: 'auth/registrations#new', as: :public_invite
-    match '/auth/finish_signup' => 'auth/confirmations#finish_signup', via: [:get, :patch], as: :finish_signup
+
+    namespace :auth do
+      resource :setup, only: [:show, :update], controller: :setup
+    end
   end
 
   devise_for :users, path: 'auth', controllers: {
diff --git a/db/seeds.rb b/db/seeds.rb
index b112cf073..0bfb5d0db 100644
--- a/db/seeds.rb
+++ b/db/seeds.rb
@@ -1,4 +1,4 @@
-Doorkeeper::Application.create!(name: 'Web', superapp: true, redirect_uri: Doorkeeper.configuration.native_redirect_uri, scopes: 'read write follow')
+Doorkeeper::Application.create!(name: 'Web', superapp: true, redirect_uri: Doorkeeper.configuration.native_redirect_uri, scopes: 'read write follow push')
 
 domain = ENV['LOCAL_DOMAIN'] || Rails.configuration.x.local_domain
 account = Account.find_or_initialize_by(id: -99, actor_type: 'Application', locked: true, username: domain)
diff --git a/spec/controllers/api/base_controller_spec.rb b/spec/controllers/api/base_controller_spec.rb
index 750ccc8cf..05a42d1c1 100644
--- a/spec/controllers/api/base_controller_spec.rb
+++ b/spec/controllers/api/base_controller_spec.rb
@@ -15,7 +15,7 @@ describe Api::BaseController do
     end
   end
 
-  describe 'Forgery protection' do
+  describe 'forgery protection' do
     before do
       routes.draw { post 'success' => 'api/base#success' }
     end
@@ -27,7 +27,45 @@ describe Api::BaseController do
     end
   end
 
-  describe 'Error handling' do
+  describe 'non-functional accounts handling' do
+    let(:user)  { Fabricate(:user, account: Fabricate(:account, username: 'alice')) }
+    let(:token) { Fabricate(:accessible_access_token, resource_owner_id: user.id, scopes: 'read') }
+
+    controller do
+      before_action :require_user!
+    end
+
+    before do
+      routes.draw { post 'success' => 'api/base#success' }
+      allow(controller).to receive(:doorkeeper_token) { token }
+    end
+
+    it 'returns http forbidden for unconfirmed accounts' do
+      user.update(confirmed_at: nil)
+      post 'success'
+      expect(response).to have_http_status(403)
+    end
+
+    it 'returns http forbidden for pending accounts' do
+      user.update(approved: false)
+      post 'success'
+      expect(response).to have_http_status(403)
+    end
+
+    it 'returns http forbidden for disabled accounts' do
+      user.update(disabled: true)
+      post 'success'
+      expect(response).to have_http_status(403)
+    end
+
+    it 'returns http forbidden for suspended accounts' do
+      user.account.suspend!
+      post 'success'
+      expect(response).to have_http_status(403)
+    end
+  end
+
+  describe 'error handling' do
     ERRORS_WITH_CODES = {
       ActiveRecord::RecordInvalid => 422,
       Mastodon::ValidationError => 422,
diff --git a/spec/controllers/application_controller_spec.rb b/spec/controllers/application_controller_spec.rb
index 27946b60f..1811500df 100644
--- a/spec/controllers/application_controller_spec.rb
+++ b/spec/controllers/application_controller_spec.rb
@@ -187,10 +187,10 @@ describe ApplicationController, type: :controller do
       expect(response).to have_http_status(200)
     end
 
-    it 'returns http 403 if user who signed in is suspended' do
+    it 'redirects to account status page' do
       sign_in(Fabricate(:user, account: Fabricate(:account, suspended: true)))
       get 'success'
-      expect(response).to have_http_status(403)
+      expect(response).to redirect_to(edit_user_registration_path)
     end
   end
 
diff --git a/spec/controllers/auth/confirmations_controller_spec.rb b/spec/controllers/auth/confirmations_controller_spec.rb
index e9a471fc5..0b6b74ff9 100644
--- a/spec/controllers/auth/confirmations_controller_spec.rb
+++ b/spec/controllers/auth/confirmations_controller_spec.rb
@@ -50,45 +50,4 @@ describe Auth::ConfirmationsController, type: :controller do
       end
     end
   end
-
-  describe 'GET #finish_signup' do
-    subject { get :finish_signup }
-
-    let(:user) { Fabricate(:user) }
-    before do
-      sign_in user, scope: :user
-      @request.env['devise.mapping'] = Devise.mappings[:user]
-    end
-
-    it 'renders finish_signup' do
-      is_expected.to render_template :finish_signup
-      expect(assigns(:user)).to have_attributes id: user.id
-    end
-  end
-
-  describe 'PATCH #finish_signup' do
-    subject { patch :finish_signup, params: { user: { email: email } } }
-
-    let(:user) { Fabricate(:user) }
-    before do
-      sign_in user, scope: :user
-      @request.env['devise.mapping'] = Devise.mappings[:user]
-    end
-
-    context 'when email is valid' do
-      let(:email) { 'new_' + user.email }
-
-      it 'redirects to root_path' do
-        is_expected.to redirect_to root_path
-      end
-    end
-
-    context 'when email is invalid' do
-      let(:email) { '' }
-
-      it 'renders finish_signup' do
-        is_expected.to render_template :finish_signup
-      end
-    end
-  end
 end
diff --git a/spec/controllers/auth/registrations_controller_spec.rb b/spec/controllers/auth/registrations_controller_spec.rb
index a4337039e..3e11b34b5 100644
--- a/spec/controllers/auth/registrations_controller_spec.rb
+++ b/spec/controllers/auth/registrations_controller_spec.rb
@@ -46,6 +46,15 @@ RSpec.describe Auth::RegistrationsController, type: :controller do
       post :update
       expect(response).to have_http_status(200)
     end
+
+    context 'when suspended' do
+      it 'returns http forbidden' do
+        request.env["devise.mapping"] = Devise.mappings[:user]
+        sign_in(Fabricate(:user, account_attributes: { username: 'test', suspended_at: Time.now.utc }), scope: :user)
+        post :update
+        expect(response).to have_http_status(403)
+      end
+    end
   end
 
   describe 'GET #new' do
@@ -94,9 +103,9 @@ RSpec.describe Auth::RegistrationsController, type: :controller do
         post :create, params: { user: { account_attributes: { username: 'test' }, email: 'test@example.com', password: '12345678', password_confirmation: '12345678' } }
       end
 
-      it 'redirects to login page' do
+      it 'redirects to setup' do
         subject
-        expect(response).to redirect_to new_user_session_path
+        expect(response).to redirect_to auth_setup_path
       end
 
       it 'creates user' do
@@ -120,9 +129,9 @@ RSpec.describe Auth::RegistrationsController, type: :controller do
         post :create, params: { user: { account_attributes: { username: 'test' }, email: 'test@example.com', password: '12345678', password_confirmation: '12345678' } }
       end
 
-      it 'redirects to login page' do
+      it 'redirects to setup' do
         subject
-        expect(response).to redirect_to new_user_session_path
+        expect(response).to redirect_to auth_setup_path
       end
 
       it 'creates user' do
@@ -148,9 +157,9 @@ RSpec.describe Auth::RegistrationsController, type: :controller do
         post :create, params: { user: { account_attributes: { username: 'test' }, email: 'test@example.com', password: '12345678', password_confirmation: '12345678', 'invite_code': invite.code } }
       end
 
-      it 'redirects to login page' do
+      it 'redirects to setup' do
         subject
-        expect(response).to redirect_to new_user_session_path
+        expect(response).to redirect_to auth_setup_path
       end
 
       it 'creates user' do
@@ -176,9 +185,9 @@ RSpec.describe Auth::RegistrationsController, type: :controller do
         post :create, params: { user: { account_attributes: { username: 'test' }, email: 'test@example.com', password: '12345678', password_confirmation: '12345678', 'invite_code': invite.code } }
       end
 
-      it 'redirects to login page' do
+      it 'redirects to setup' do
         subject
-        expect(response).to redirect_to new_user_session_path
+        expect(response).to redirect_to auth_setup_path
       end
 
       it 'creates user' do
diff --git a/spec/controllers/auth/sessions_controller_spec.rb b/spec/controllers/auth/sessions_controller_spec.rb
index 71fcc1a6e..87ef4f2bb 100644
--- a/spec/controllers/auth/sessions_controller_spec.rb
+++ b/spec/controllers/auth/sessions_controller_spec.rb
@@ -160,8 +160,8 @@ RSpec.describe Auth::SessionsController, type: :controller do
         let(:unconfirmed_user) { user.tap { |u| u.update!(confirmed_at: nil) } }
         let(:accept_language) { 'fr' }
 
-        it 'shows a translated login error' do
-          expect(flash[:alert]).to eq(I18n.t('devise.failure.unconfirmed', locale: accept_language))
+        it 'redirects to home' do
+          expect(response).to redirect_to(root_path)
         end
       end
 
diff --git a/spec/controllers/settings/deletes_controller_spec.rb b/spec/controllers/settings/deletes_controller_spec.rb
index 35fd64e9b..996872efd 100644
--- a/spec/controllers/settings/deletes_controller_spec.rb
+++ b/spec/controllers/settings/deletes_controller_spec.rb
@@ -15,6 +15,15 @@ describe Settings::DeletesController do
         get :show
         expect(response).to have_http_status(200)
       end
+
+      context 'when suspended' do
+        let(:user) { Fabricate(:user, account_attributes: { username: 'alice', suspended_at: Time.now.utc }) }
+
+        it 'returns http forbidden' do
+          get :show
+          expect(response).to have_http_status(403)
+        end
+      end
     end
 
     context 'when not signed in' do
@@ -49,6 +58,14 @@ describe Settings::DeletesController do
         it 'marks account as suspended' do
           expect(user.account.reload).to be_suspended
         end
+
+        context 'when suspended' do
+          let(:user) { Fabricate(:user, account_attributes: { username: 'alice', suspended_at: Time.now.utc }) }
+
+          it 'returns http forbidden' do
+            expect(response).to have_http_status(403)
+          end
+        end
       end
 
       context 'with incorrect password' do
diff --git a/spec/features/log_in_spec.rb b/spec/features/log_in_spec.rb
index 53a1f9b12..f6c26cd0f 100644
--- a/spec/features/log_in_spec.rb
+++ b/spec/features/log_in_spec.rb
@@ -31,12 +31,12 @@ feature "Log in" do
   context do
     given(:confirmed_at) { nil }
 
-    scenario "A unconfirmed user is not able to log in" do
+    scenario "A unconfirmed user is able to log in" do
       fill_in "user_email", with: email
       fill_in "user_password", with: password
       click_on I18n.t('auth.login')
 
-      is_expected.to have_css(".flash-message", text: failure_message("unconfirmed"))
+      is_expected.to have_css("div.admin-wrapper")
     end
   end
 
diff --git a/spec/models/user_spec.rb b/spec/models/user_spec.rb
index 856254ce4..d7c0b5359 100644
--- a/spec/models/user_spec.rb
+++ b/spec/models/user_spec.rb
@@ -506,7 +506,7 @@ RSpec.describe User, type: :model do
       context 'when user is not confirmed' do
         let(:confirmed_at) { nil }
 
-        it { is_expected.to be false }
+        it { is_expected.to be true }
       end
     end
 
@@ -522,7 +522,7 @@ RSpec.describe User, type: :model do
       context 'when user is not confirmed' do
         let(:confirmed_at) { nil }
 
-        it { is_expected.to be false }
+        it { is_expected.to be true }
       end
     end
   end
-- 
cgit 


From 92569ffde81a3fe768fd8773fdab5d1a927f3f88 Mon Sep 17 00:00:00 2001
From: ThibG <thib@sitedethib.com>
Date: Fri, 26 Jul 2019 18:55:33 +0200
Subject: Fix invites not being disabled upon account suspension (#11412)

* Disable invite links from disabled/suspended users

* Add has_many invites relationship to users

* Destroy unused invites when suspending an account
---
 app/controllers/invites_controller.rb   |  2 +-
 app/models/invite.rb                    |  4 ++--
 app/models/user.rb                      |  1 +
 app/services/suspend_account_service.rb |  1 +
 spec/models/invite_spec.rb              | 16 +++++++++++-----
 5 files changed, 16 insertions(+), 8 deletions(-)

(limited to 'app/models')

diff --git a/app/controllers/invites_controller.rb b/app/controllers/invites_controller.rb
index fdb3a0962..de5280305 100644
--- a/app/controllers/invites_controller.rb
+++ b/app/controllers/invites_controller.rb
@@ -39,7 +39,7 @@ class InvitesController < ApplicationController
   private
 
   def invites
-    Invite.where(user: current_user).order(id: :desc)
+    current_user.invites.order(id: :desc)
   end
 
   def resource_params
diff --git a/app/models/invite.rb b/app/models/invite.rb
index fe2322462..02ab8e0b2 100644
--- a/app/models/invite.rb
+++ b/app/models/invite.rb
@@ -17,7 +17,7 @@
 class Invite < ApplicationRecord
   include Expireable
 
-  belongs_to :user
+  belongs_to :user, inverse_of: :invites
   has_many :users, inverse_of: :invite
 
   scope :available, -> { where(expires_at: nil).or(where('expires_at >= ?', Time.now.utc)) }
@@ -25,7 +25,7 @@ class Invite < ApplicationRecord
   before_validation :set_code
 
   def valid_for_use?
-    (max_uses.nil? || uses < max_uses) && !expired?
+    (max_uses.nil? || uses < max_uses) && !expired? && !(user.nil? || user.disabled?)
   end
 
   private
diff --git a/app/models/user.rb b/app/models/user.rb
index 474c77293..6806c0362 100644
--- a/app/models/user.rb
+++ b/app/models/user.rb
@@ -73,6 +73,7 @@ class User < ApplicationRecord
 
   has_many :applications, class_name: 'Doorkeeper::Application', as: :owner
   has_many :backups, inverse_of: :user
+  has_many :invites, inverse_of: :user
 
   has_one :invite_request, class_name: 'UserInviteRequest', inverse_of: :user, dependent: :destroy
   accepts_nested_attributes_for :invite_request, reject_if: ->(attributes) { attributes['text'].blank? }
diff --git a/app/services/suspend_account_service.rb b/app/services/suspend_account_service.rb
index 00cffcdfc..902af376c 100644
--- a/app/services/suspend_account_service.rb
+++ b/app/services/suspend_account_service.rb
@@ -64,6 +64,7 @@ class SuspendAccountService < BaseService
       @account.user.destroy
     else
       @account.user.disable!
+      @account.user.invites.where(uses: 0).destroy_all
     end
   end
 
diff --git a/spec/models/invite_spec.rb b/spec/models/invite_spec.rb
index 0ba1dccb3..30abfb86b 100644
--- a/spec/models/invite_spec.rb
+++ b/spec/models/invite_spec.rb
@@ -3,27 +3,33 @@ require 'rails_helper'
 RSpec.describe Invite, type: :model do
   describe '#valid_for_use?' do
     it 'returns true when there are no limitations' do
-      invite = Invite.new(max_uses: nil, expires_at: nil)
+      invite = Fabricate(:invite, max_uses: nil, expires_at: nil)
       expect(invite.valid_for_use?).to be true
     end
 
     it 'returns true when not expired' do
-      invite = Invite.new(max_uses: nil, expires_at: 1.hour.from_now)
+      invite = Fabricate(:invite, max_uses: nil, expires_at: 1.hour.from_now)
       expect(invite.valid_for_use?).to be true
     end
 
     it 'returns false when expired' do
-      invite = Invite.new(max_uses: nil, expires_at: 1.hour.ago)
+      invite = Fabricate(:invite, max_uses: nil, expires_at: 1.hour.ago)
       expect(invite.valid_for_use?).to be false
     end
 
     it 'returns true when uses still available' do
-      invite = Invite.new(max_uses: 250, uses: 249, expires_at: nil)
+      invite = Fabricate(:invite, max_uses: 250, uses: 249, expires_at: nil)
       expect(invite.valid_for_use?).to be true
     end
 
     it 'returns false when maximum uses reached' do
-      invite = Invite.new(max_uses: 250, uses: 250, expires_at: nil)
+      invite = Fabricate(:invite, max_uses: 250, uses: 250, expires_at: nil)
+      expect(invite.valid_for_use?).to be false
+    end
+
+    it 'returns false when invite creator has been disabled' do
+      invite = Fabricate(:invite, max_uses: nil, expires_at: nil)
+      SuspendAccountService.new.call(invite.user.account)
       expect(invite.valid_for_use?).to be false
     end
   end
-- 
cgit 


From 501148ab912b3bd36dbf0f9f2e10bfde7787012d Mon Sep 17 00:00:00 2001
From: Eugen Rochko <eugen@zeonfederated.com>
Date: Sat, 27 Jul 2019 04:41:55 +0200
Subject: Remove timestamps from converted images to make them deterministic
 (#11408)

---
 app/models/media_attachment.rb | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

(limited to 'app/models')

diff --git a/app/models/media_attachment.rb b/app/models/media_attachment.rb
index 05be302af..2fc5ccf27 100644
--- a/app/models/media_attachment.rb
+++ b/app/models/media_attachment.rb
@@ -113,7 +113,7 @@ class MediaAttachment < ApplicationRecord
   has_attached_file :file,
                     styles: ->(f) { file_styles f },
                     processors: ->(f) { file_processors f },
-                    convert_options: { all: '-quality 90 -strip' }
+                    convert_options: { all: '-quality 90 -strip +set modify-date +set create-date' }
 
   validates_attachment_content_type :file, content_type: IMAGE_MIME_TYPES + VIDEO_MIME_TYPES + AUDIO_MIME_TYPES
   validates_attachment_size :file, less_than: IMAGE_LIMIT, unless: :larger_media_format?
-- 
cgit 


From f371b32137ccd7e74ca29d25af2072fb79654b15 Mon Sep 17 00:00:00 2001
From: Eugen Rochko <eugen@zeonfederated.com>
Date: Sun, 28 Jul 2019 05:59:51 +0200
Subject: Change hashtags to preserve first-used casing (#11416)

---
 app/lib/activitypub/activity/create.rb             |  9 ++----
 app/models/tag.rb                                  | 34 +++++++++++++++++++---
 app/services/hashtag_query_service.rb              |  4 +--
 app/services/process_hashtags_service.rb           |  4 +--
 ...726175042_add_case_insensitive_index_to_tags.rb | 15 ++++++++++
 db/schema.rb                                       |  5 ++--
 6 files changed, 53 insertions(+), 18 deletions(-)
 create mode 100644 db/migrate/20190726175042_add_case_insensitive_index_to_tags.rb

(limited to 'app/models')

diff --git a/app/lib/activitypub/activity/create.rb b/app/lib/activitypub/activity/create.rb
index 56c24680a..000b77df5 100644
--- a/app/lib/activitypub/activity/create.rb
+++ b/app/lib/activitypub/activity/create.rb
@@ -148,12 +148,9 @@ class ActivityPub::Activity::Create < ActivityPub::Activity
   def process_hashtag(tag)
     return if tag['name'].blank?
 
-    hashtag = tag['name'].gsub(/\A#/, '').mb_chars.downcase
-    hashtag = Tag.where(name: hashtag).first_or_create!(name: hashtag)
-
-    return if @tags.include?(hashtag)
-
-    @tags << hashtag
+    Tag.find_or_create_by_names(tag['name']) do |hashtag|
+      @tags << hashtag unless @tags.include?(hashtag)
+    end
   rescue ActiveRecord::RecordInvalid
     nil
   end
diff --git a/app/models/tag.rb b/app/models/tag.rb
index b371d59c1..972242064 100644
--- a/app/models/tag.rb
+++ b/app/models/tag.rb
@@ -20,7 +20,7 @@ class Tag < ApplicationRecord
   HASHTAG_NAME_RE = '([[:word:]_][[:word:]_·]*[[:alpha:]_·][[:word:]_·]*[[:word:]_])|([[:word:]_]*[[:alpha:]][[:word:]_]*)'
   HASHTAG_RE = /(?:^|[^\/\)\w])#(#{HASHTAG_NAME_RE})/i
 
-  validates :name, presence: true, uniqueness: true, format: { with: /\A(#{HASHTAG_NAME_RE})\z/i }
+  validates :name, presence: true, format: { with: /\A(#{HASHTAG_NAME_RE})\z/i }
 
   scope :discoverable, -> { joins(:account_tag_stat).where(AccountTagStat.arel_table[:accounts_count].gt(0)).where(account_tag_stats: { hidden: false }).order(Arel.sql('account_tag_stats.accounts_count desc')) }
   scope :hidden, -> { where(account_tag_stats: { hidden: true }) }
@@ -64,22 +64,48 @@ class Tag < ApplicationRecord
   end
 
   class << self
+    def find_or_create_by_names(name_or_names)
+      Array(name_or_names).map(&method(:normalize)).uniq.map do |normalized_name|
+        tag = matching_name(normalized_name).first || create(name: normalized_name)
+
+        yield tag if block_given?
+
+        tag
+      end
+    end
+
     def search_for(term, limit = 5, offset = 0)
-      pattern = sanitize_sql_like(term.strip) + '%'
+      pattern = sanitize_sql_like(normalize(term.strip)) + '%'
 
-      Tag.where('lower(name) like lower(?)', pattern)
+      Tag.where(arel_table[:name].lower.matches(pattern.downcase))
          .order(:name)
          .limit(limit)
          .offset(offset)
     end
 
     def find_normalized(name)
-      find_by(name: name.mb_chars.downcase.to_s)
+      matching_name(name).first
     end
 
     def find_normalized!(name)
       find_normalized(name) || raise(ActiveRecord::RecordNotFound)
     end
+
+    def matching_name(name_or_names)
+      names = Array(name_or_names).map { |name| normalize(name).downcase }
+
+      if names.size == 1
+        where(arel_table[:name].lower.eq(names.first))
+      else
+        where(arel_table[:name].lower.in(names))
+      end
+    end
+
+    private
+
+    def normalize(str)
+      str.gsub(/\A#/, '').mb_chars.to_s
+    end
   end
 
   private
diff --git a/app/services/hashtag_query_service.rb b/app/services/hashtag_query_service.rb
index 5773d78c6..282821710 100644
--- a/app/services/hashtag_query_service.rb
+++ b/app/services/hashtag_query_service.rb
@@ -14,7 +14,7 @@ class HashtagQueryService < BaseService
 
   private
 
-  def tags_for(tags)
-    Tag.where(name: tags.map(&:downcase)) if tags.presence
+  def tags_for(names)
+    Tag.matching_name(names) if names.presence
   end
 end
diff --git a/app/services/process_hashtags_service.rb b/app/services/process_hashtags_service.rb
index b6974e598..e8e139b05 100644
--- a/app/services/process_hashtags_service.rb
+++ b/app/services/process_hashtags_service.rb
@@ -5,9 +5,7 @@ class ProcessHashtagsService < BaseService
     tags    = Extractor.extract_hashtags(status.text) if status.local?
     records = []
 
-    tags.map { |str| str.mb_chars.downcase }.uniq(&:to_s).each do |name|
-      tag = Tag.where(name: name).first_or_create(name: name)
-
+    Tag.find_or_create_by_names(tags) do |tag|
       status.tags << tag
       records << tag
 
diff --git a/db/migrate/20190726175042_add_case_insensitive_index_to_tags.rb b/db/migrate/20190726175042_add_case_insensitive_index_to_tags.rb
new file mode 100644
index 000000000..6fa8c0ec4
--- /dev/null
+++ b/db/migrate/20190726175042_add_case_insensitive_index_to_tags.rb
@@ -0,0 +1,15 @@
+class AddCaseInsensitiveIndexToTags < ActiveRecord::Migration[5.2]
+  disable_ddl_transaction!
+
+  def up
+    safety_assured { execute 'CREATE UNIQUE INDEX CONCURRENTLY index_tags_on_name_lower ON tags (lower(name))' }
+    remove_index :tags, name: 'index_tags_on_name'
+    remove_index :tags, name: 'hashtag_search_index'
+  end
+
+  def down
+    add_index :tags, :name, unique: true, algorithm: :concurrently
+    safety_assured { execute 'CREATE INDEX CONCURRENTLY hashtag_search_index ON tags (name text_pattern_ops)' }
+    remove_index :tags, name: 'index_tags_on_name_lower'
+  end
+end
diff --git a/db/schema.rb b/db/schema.rb
index 6319dd932..1847305c7 100644
--- a/db/schema.rb
+++ b/db/schema.rb
@@ -10,7 +10,7 @@
 #
 # It's strongly recommended that you check this file into your version control system.
 
-ActiveRecord::Schema.define(version: 2019_07_15_164535) do
+ActiveRecord::Schema.define(version: 2019_07_26_175042) do
 
   # These are extensions that must be enabled in order to support this database
   enable_extension "plpgsql"
@@ -652,8 +652,7 @@ ActiveRecord::Schema.define(version: 2019_07_15_164535) do
     t.string "name", default: "", null: false
     t.datetime "created_at", null: false
     t.datetime "updated_at", null: false
-    t.index "lower((name)::text) text_pattern_ops", name: "hashtag_search_index"
-    t.index ["name"], name: "index_tags_on_name", unique: true
+    t.index "lower((name)::text)", name: "index_tags_on_name_lower", unique: true
   end
 
   create_table "tombstones", force: :cascade do |t|
-- 
cgit 


From 75f7f9930eb2a6f5c4041ec44fe0aa795c9ec449 Mon Sep 17 00:00:00 2001
From: Eugen Rochko <eugen@zeonfederated.com>
Date: Sun, 28 Jul 2019 17:30:12 +0200
Subject: Remove conversation URI (#11423)

It is not part of ActivityPub and will free up a lot of space
---
 app/lib/activitypub/activity/create.rb                    | 11 -----------
 app/models/conversation.rb                                |  7 -------
 app/serializers/activitypub/note_serializer.rb            | 15 ++-------------
 .../20190519130537_remove_boosts_widening_audience.rb     |  2 ++
 .../20190728084117_remove_uri_from_conversations.rb       | 12 ++++++++++++
 db/schema.rb                                              |  4 +---
 spec/models/conversation_spec.rb                          |  8 --------
 7 files changed, 17 insertions(+), 42 deletions(-)
 create mode 100644 db/post_migrate/20190728084117_remove_uri_from_conversations.rb

(limited to 'app/models')

diff --git a/app/lib/activitypub/activity/create.rb b/app/lib/activitypub/activity/create.rb
index 000b77df5..eb7b09e44 100644
--- a/app/lib/activitypub/activity/create.rb
+++ b/app/lib/activitypub/activity/create.rb
@@ -67,7 +67,6 @@ class ActivityPub::Activity::Create < ActivityPub::Activity
         sensitive: @object['sensitive'] || false,
         visibility: visibility_from_audience,
         thread: replied_to_status,
-        conversation: conversation_from_uri(@object['conversation']),
         media_attachment_ids: process_attachments.take(4).map(&:id),
         poll: process_poll,
       }
@@ -262,16 +261,6 @@ class ActivityPub::Activity::Create < ActivityPub::Activity
     ActivityPub::FetchRepliesWorker.perform_async(status.id, uri) unless uri.nil?
   end
 
-  def conversation_from_uri(uri)
-    return nil if uri.nil?
-    return Conversation.find_by(id: OStatus::TagManager.instance.unique_tag_to_local_id(uri, 'Conversation')) if OStatus::TagManager.instance.local_id?(uri)
-    begin
-      Conversation.find_or_create_by!(uri: uri)
-    rescue ActiveRecord::RecordInvalid, ActiveRecord::RecordNotUnique
-      retry
-    end
-  end
-
   def visibility_from_audience
     if equals_or_includes?(@object['to'], ActivityPub::TagManager::COLLECTIONS[:public])
       :public
diff --git a/app/models/conversation.rb b/app/models/conversation.rb
index 4dfaea889..37d233f32 100644
--- a/app/models/conversation.rb
+++ b/app/models/conversation.rb
@@ -4,17 +4,10 @@
 # Table name: conversations
 #
 #  id         :bigint(8)        not null, primary key
-#  uri        :string
 #  created_at :datetime         not null
 #  updated_at :datetime         not null
 #
 
 class Conversation < ApplicationRecord
-  validates :uri, uniqueness: true, if: :uri?
-
   has_many :statuses
-
-  def local?
-    uri.nil?
-  end
 end
diff --git a/app/serializers/activitypub/note_serializer.rb b/app/serializers/activitypub/note_serializer.rb
index 67f596e78..88b50b3ea 100644
--- a/app/serializers/activitypub/note_serializer.rb
+++ b/app/serializers/activitypub/note_serializer.rb
@@ -1,14 +1,13 @@
 # frozen_string_literal: true
 
 class ActivityPub::NoteSerializer < ActivityPub::Serializer
-  context_extensions :atom_uri, :conversation, :sensitive,
+  context_extensions :atom_uri, :sensitive,
                      :hashtag, :emoji, :focal_point, :blurhash
 
   attributes :id, :type, :summary,
              :in_reply_to, :published, :url,
              :attributed_to, :to, :cc, :sensitive,
-             :atom_uri, :in_reply_to_atom_uri,
-             :conversation
+             :atom_uri, :in_reply_to_atom_uri
 
   attribute :content
   attribute :content_map, if: :language?
@@ -110,16 +109,6 @@ class ActivityPub::NoteSerializer < ActivityPub::Serializer
     OStatus::TagManager.instance.uri_for(object.thread)
   end
 
-  def conversation
-    return if object.conversation.nil?
-
-    if object.conversation.uri?
-      object.conversation.uri
-    else
-      OStatus::TagManager.instance.unique_tag(object.conversation.created_at, object.conversation.id, 'Conversation')
-    end
-  end
-
   def local?
     object.account.local?
   end
diff --git a/db/post_migrate/20190519130537_remove_boosts_widening_audience.rb b/db/post_migrate/20190519130537_remove_boosts_widening_audience.rb
index d2d924239..b89efd989 100644
--- a/db/post_migrate/20190519130537_remove_boosts_widening_audience.rb
+++ b/db/post_migrate/20190519130537_remove_boosts_widening_audience.rb
@@ -1,3 +1,5 @@
+# frozen_string_literal: true
+
 class RemoveBoostsWideningAudience < ActiveRecord::Migration[5.2]
   disable_ddl_transaction!
 
diff --git a/db/post_migrate/20190728084117_remove_uri_from_conversations.rb b/db/post_migrate/20190728084117_remove_uri_from_conversations.rb
new file mode 100644
index 000000000..b2b157ef7
--- /dev/null
+++ b/db/post_migrate/20190728084117_remove_uri_from_conversations.rb
@@ -0,0 +1,12 @@
+# frozen_string_literal: true
+
+class RemoveUriFromConversations < ActiveRecord::Migration[5.2]
+  def up
+    safety_assured { remove_column :conversations, :uri, :string }
+  end
+
+  def down
+    add_column :conversations, :uri, :string
+    add_index :conversations, :uri, unique: true
+  end
+end
diff --git a/db/schema.rb b/db/schema.rb
index 1847305c7..a722bad36 100644
--- a/db/schema.rb
+++ b/db/schema.rb
@@ -10,7 +10,7 @@
 #
 # It's strongly recommended that you check this file into your version control system.
 
-ActiveRecord::Schema.define(version: 2019_07_26_175042) do
+ActiveRecord::Schema.define(version: 2019_07_28_084117) do
 
   # These are extensions that must be enabled in order to support this database
   enable_extension "plpgsql"
@@ -203,10 +203,8 @@ ActiveRecord::Schema.define(version: 2019_07_26_175042) do
   end
 
   create_table "conversations", force: :cascade do |t|
-    t.string "uri"
     t.datetime "created_at", null: false
     t.datetime "updated_at", null: false
-    t.index ["uri"], name: "index_conversations_on_uri", unique: true
   end
 
   create_table "custom_emoji_categories", force: :cascade do |t|
diff --git a/spec/models/conversation_spec.rb b/spec/models/conversation_spec.rb
index 8b5e4fdaf..b91d36169 100644
--- a/spec/models/conversation_spec.rb
+++ b/spec/models/conversation_spec.rb
@@ -1,13 +1,5 @@
 require 'rails_helper'
 
 RSpec.describe Conversation, type: :model do
-  describe '#local?' do
-    it 'returns true when URI is nil' do
-      expect(Fabricate(:conversation).local?).to be true
-    end
 
-    it 'returns false when URI is not nil' do
-      expect(Fabricate(:conversation, uri: 'abc').local?).to be false
-    end
-  end
 end
-- 
cgit 


From b9b0313c78ca4efcc8cda089c652be0b333aa16a Mon Sep 17 00:00:00 2001
From: Eugen Rochko <eugen@zeonfederated.com>
Date: Sun, 28 Jul 2019 17:47:37 +0200
Subject: Revert "Remove conversation URI (#11423)" (#11424)

This reverts commit 75f7f9930eb2a6f5c4041ec44fe0aa795c9ec449.
---
 app/lib/activitypub/activity/create.rb                    | 11 +++++++++++
 app/models/conversation.rb                                |  7 +++++++
 app/serializers/activitypub/note_serializer.rb            | 15 +++++++++++++--
 .../20190519130537_remove_boosts_widening_audience.rb     |  2 --
 .../20190728084117_remove_uri_from_conversations.rb       | 12 ------------
 db/schema.rb                                              |  4 +++-
 spec/models/conversation_spec.rb                          |  8 ++++++++
 7 files changed, 42 insertions(+), 17 deletions(-)
 delete mode 100644 db/post_migrate/20190728084117_remove_uri_from_conversations.rb

(limited to 'app/models')

diff --git a/app/lib/activitypub/activity/create.rb b/app/lib/activitypub/activity/create.rb
index eb7b09e44..000b77df5 100644
--- a/app/lib/activitypub/activity/create.rb
+++ b/app/lib/activitypub/activity/create.rb
@@ -67,6 +67,7 @@ class ActivityPub::Activity::Create < ActivityPub::Activity
         sensitive: @object['sensitive'] || false,
         visibility: visibility_from_audience,
         thread: replied_to_status,
+        conversation: conversation_from_uri(@object['conversation']),
         media_attachment_ids: process_attachments.take(4).map(&:id),
         poll: process_poll,
       }
@@ -261,6 +262,16 @@ class ActivityPub::Activity::Create < ActivityPub::Activity
     ActivityPub::FetchRepliesWorker.perform_async(status.id, uri) unless uri.nil?
   end
 
+  def conversation_from_uri(uri)
+    return nil if uri.nil?
+    return Conversation.find_by(id: OStatus::TagManager.instance.unique_tag_to_local_id(uri, 'Conversation')) if OStatus::TagManager.instance.local_id?(uri)
+    begin
+      Conversation.find_or_create_by!(uri: uri)
+    rescue ActiveRecord::RecordInvalid, ActiveRecord::RecordNotUnique
+      retry
+    end
+  end
+
   def visibility_from_audience
     if equals_or_includes?(@object['to'], ActivityPub::TagManager::COLLECTIONS[:public])
       :public
diff --git a/app/models/conversation.rb b/app/models/conversation.rb
index 37d233f32..4dfaea889 100644
--- a/app/models/conversation.rb
+++ b/app/models/conversation.rb
@@ -4,10 +4,17 @@
 # Table name: conversations
 #
 #  id         :bigint(8)        not null, primary key
+#  uri        :string
 #  created_at :datetime         not null
 #  updated_at :datetime         not null
 #
 
 class Conversation < ApplicationRecord
+  validates :uri, uniqueness: true, if: :uri?
+
   has_many :statuses
+
+  def local?
+    uri.nil?
+  end
 end
diff --git a/app/serializers/activitypub/note_serializer.rb b/app/serializers/activitypub/note_serializer.rb
index 88b50b3ea..67f596e78 100644
--- a/app/serializers/activitypub/note_serializer.rb
+++ b/app/serializers/activitypub/note_serializer.rb
@@ -1,13 +1,14 @@
 # frozen_string_literal: true
 
 class ActivityPub::NoteSerializer < ActivityPub::Serializer
-  context_extensions :atom_uri, :sensitive,
+  context_extensions :atom_uri, :conversation, :sensitive,
                      :hashtag, :emoji, :focal_point, :blurhash
 
   attributes :id, :type, :summary,
              :in_reply_to, :published, :url,
              :attributed_to, :to, :cc, :sensitive,
-             :atom_uri, :in_reply_to_atom_uri
+             :atom_uri, :in_reply_to_atom_uri,
+             :conversation
 
   attribute :content
   attribute :content_map, if: :language?
@@ -109,6 +110,16 @@ class ActivityPub::NoteSerializer < ActivityPub::Serializer
     OStatus::TagManager.instance.uri_for(object.thread)
   end
 
+  def conversation
+    return if object.conversation.nil?
+
+    if object.conversation.uri?
+      object.conversation.uri
+    else
+      OStatus::TagManager.instance.unique_tag(object.conversation.created_at, object.conversation.id, 'Conversation')
+    end
+  end
+
   def local?
     object.account.local?
   end
diff --git a/db/post_migrate/20190519130537_remove_boosts_widening_audience.rb b/db/post_migrate/20190519130537_remove_boosts_widening_audience.rb
index b89efd989..d2d924239 100644
--- a/db/post_migrate/20190519130537_remove_boosts_widening_audience.rb
+++ b/db/post_migrate/20190519130537_remove_boosts_widening_audience.rb
@@ -1,5 +1,3 @@
-# frozen_string_literal: true
-
 class RemoveBoostsWideningAudience < ActiveRecord::Migration[5.2]
   disable_ddl_transaction!
 
diff --git a/db/post_migrate/20190728084117_remove_uri_from_conversations.rb b/db/post_migrate/20190728084117_remove_uri_from_conversations.rb
deleted file mode 100644
index b2b157ef7..000000000
--- a/db/post_migrate/20190728084117_remove_uri_from_conversations.rb
+++ /dev/null
@@ -1,12 +0,0 @@
-# frozen_string_literal: true
-
-class RemoveUriFromConversations < ActiveRecord::Migration[5.2]
-  def up
-    safety_assured { remove_column :conversations, :uri, :string }
-  end
-
-  def down
-    add_column :conversations, :uri, :string
-    add_index :conversations, :uri, unique: true
-  end
-end
diff --git a/db/schema.rb b/db/schema.rb
index a722bad36..1847305c7 100644
--- a/db/schema.rb
+++ b/db/schema.rb
@@ -10,7 +10,7 @@
 #
 # It's strongly recommended that you check this file into your version control system.
 
-ActiveRecord::Schema.define(version: 2019_07_28_084117) do
+ActiveRecord::Schema.define(version: 2019_07_26_175042) do
 
   # These are extensions that must be enabled in order to support this database
   enable_extension "plpgsql"
@@ -203,8 +203,10 @@ ActiveRecord::Schema.define(version: 2019_07_28_084117) do
   end
 
   create_table "conversations", force: :cascade do |t|
+    t.string "uri"
     t.datetime "created_at", null: false
     t.datetime "updated_at", null: false
+    t.index ["uri"], name: "index_conversations_on_uri", unique: true
   end
 
   create_table "custom_emoji_categories", force: :cascade do |t|
diff --git a/spec/models/conversation_spec.rb b/spec/models/conversation_spec.rb
index b91d36169..8b5e4fdaf 100644
--- a/spec/models/conversation_spec.rb
+++ b/spec/models/conversation_spec.rb
@@ -1,5 +1,13 @@
 require 'rails_helper'
 
 RSpec.describe Conversation, type: :model do
+  describe '#local?' do
+    it 'returns true when URI is nil' do
+      expect(Fabricate(:conversation).local?).to be true
+    end
 
+    it 'returns false when URI is not nil' do
+      expect(Fabricate(:conversation, uri: 'abc').local?).to be false
+    end
+  end
 end
-- 
cgit 


From e136112ab76a37cbde5c1bcfeb562c8e0a5b5116 Mon Sep 17 00:00:00 2001
From: Eugen Rochko <eugen@zeonfederated.com>
Date: Mon, 29 Jul 2019 20:40:21 +0200
Subject: Fix tag normalization and migration not removing duplicate tags
 (#11441)

Fix #11428
---
 app/models/tag.rb                                  |  8 ++---
 ...726175042_add_case_insensitive_index_to_tags.rb | 13 +++++++++
 spec/models/tag_spec.rb                            | 34 ++++++++++++++++++++++
 3 files changed, 51 insertions(+), 4 deletions(-)

(limited to 'app/models')

diff --git a/app/models/tag.rb b/app/models/tag.rb
index 972242064..46e3a3ec0 100644
--- a/app/models/tag.rb
+++ b/app/models/tag.rb
@@ -65,7 +65,7 @@ class Tag < ApplicationRecord
 
   class << self
     def find_or_create_by_names(name_or_names)
-      Array(name_or_names).map(&method(:normalize)).uniq.map do |normalized_name|
+      Array(name_or_names).map(&method(:normalize)).uniq { |str| str.mb_chars.downcase.to_s }.map do |normalized_name|
         tag = matching_name(normalized_name).first || create(name: normalized_name)
 
         yield tag if block_given?
@@ -77,7 +77,7 @@ class Tag < ApplicationRecord
     def search_for(term, limit = 5, offset = 0)
       pattern = sanitize_sql_like(normalize(term.strip)) + '%'
 
-      Tag.where(arel_table[:name].lower.matches(pattern.downcase))
+      Tag.where(arel_table[:name].lower.matches(pattern.mb_chars.downcase.to_s))
          .order(:name)
          .limit(limit)
          .offset(offset)
@@ -92,7 +92,7 @@ class Tag < ApplicationRecord
     end
 
     def matching_name(name_or_names)
-      names = Array(name_or_names).map { |name| normalize(name).downcase }
+      names = Array(name_or_names).map { |name| normalize(name).mb_chars.downcase.to_s }
 
       if names.size == 1
         where(arel_table[:name].lower.eq(names.first))
@@ -104,7 +104,7 @@ class Tag < ApplicationRecord
     private
 
     def normalize(str)
-      str.gsub(/\A#/, '').mb_chars.to_s
+      str.gsub(/\A#/, '')
     end
   end
 
diff --git a/db/migrate/20190726175042_add_case_insensitive_index_to_tags.rb b/db/migrate/20190726175042_add_case_insensitive_index_to_tags.rb
index 6fa8c0ec4..057fc86ba 100644
--- a/db/migrate/20190726175042_add_case_insensitive_index_to_tags.rb
+++ b/db/migrate/20190726175042_add_case_insensitive_index_to_tags.rb
@@ -2,6 +2,19 @@ class AddCaseInsensitiveIndexToTags < ActiveRecord::Migration[5.2]
   disable_ddl_transaction!
 
   def up
+    Tag.connection.select_all('SELECT string_agg(id::text, \',\') AS ids FROM tags GROUP BY lower(name) HAVING count(*) > 1').to_hash.each do |row|
+      canonical_tag_id  = row['ids'].split(',').first
+      redundant_tag_ids = row['ids'].split(',')[1..-1]
+
+      safety_assured do
+        execute "UPDATE accounts_tags AS t0 SET tag_id = #{canonical_tag_id} WHERE tag_id IN (#{redundant_tag_ids.join(', ')}) AND NOT EXISTS (SELECT t1.tag_id FROM accounts_tags AS t1 WHERE t1.tag_id = #{canonical_tag_id} AND t1.account_id = t0.account_id)"
+        execute "UPDATE statuses_tags AS t0 SET tag_id = #{canonical_tag_id} WHERE tag_id IN (#{redundant_tag_ids.join(', ')}) AND NOT EXISTS (SELECT t1.tag_id FROM statuses_tags AS t1 WHERE t1.tag_id = #{canonical_tag_id} AND t1.status_id = t0.status_id)"
+        execute "UPDATE featured_tags AS t0 SET tag_id = #{canonical_tag_id} WHERE tag_id IN (#{redundant_tag_ids.join(', ')})  AND NOT EXISTS (SELECT t1.tag_id FROM featured_tags AS t1 WHERE t1.tag_id = #{canonical_tag_id} AND t1.account_id = t0.account_id)"
+      end
+
+      Tag.where(id: redundant_tag_ids).in_batches.delete_all
+    end
+
     safety_assured { execute 'CREATE UNIQUE INDEX CONCURRENTLY index_tags_on_name_lower ON tags (lower(name))' }
     remove_index :tags, name: 'index_tags_on_name'
     remove_index :tags, name: 'hashtag_search_index'
diff --git a/spec/models/tag_spec.rb b/spec/models/tag_spec.rb
index 9a30ceaa5..5f07fd618 100644
--- a/spec/models/tag_spec.rb
+++ b/spec/models/tag_spec.rb
@@ -82,6 +82,40 @@ RSpec.describe Tag, type: :model do
     end
   end
 
+  describe '.find_normalized' do
+    it 'returns tag for a multibyte case-insensitive name' do
+      upcase_string   = 'abcABCａｂｃＡＢＣやゆよ'
+      downcase_string = 'abcabcａｂｃａｂｃやゆよ';
+
+      tag = Fabricate(:tag, name: downcase_string)
+      expect(Tag.find_normalized(upcase_string)).to eq tag
+    end
+  end
+
+  describe '.matching_name' do
+    it 'returns tags for multibyte case-insensitive names' do
+      upcase_string   = 'abcABCａｂｃＡＢＣやゆよ'
+      downcase_string = 'abcabcａｂｃａｂｃやゆよ';
+
+      tag = Fabricate(:tag, name: downcase_string)
+      expect(Tag.matching_name(upcase_string)).to eq [tag]
+    end
+  end
+
+  describe '.find_or_create_by_names' do
+    it 'runs a passed block once per tag regardless of duplicates' do
+      upcase_string   = 'abcABCａｂｃＡＢＣやゆよ'
+      downcase_string = 'abcabcａｂｃａｂｃやゆよ';
+      count           = 0
+
+      Tag.find_or_create_by_names([upcase_string, downcase_string]) do |tag|
+        count += 1
+      end
+
+      expect(count).to eq 1
+    end
+  end
+
   describe '.search_for' do
     it 'finds tag records with matching names' do
       tag = Fabricate(:tag, name: "match")
-- 
cgit 


From 24552b5160a5090e7d6056fb69a209aa48fe4fce Mon Sep 17 00:00:00 2001
From: Eugen Rochko <eugen@zeonfederated.com>
Date: Tue, 30 Jul 2019 11:10:46 +0200
Subject: Add whitelist mode (#11291)

---
 app/controllers/about_controller.rb                |  5 +++
 app/controllers/activitypub/base_controller.rb     |  2 ++
 app/controllers/activitypub/inboxes_controller.rb  |  2 +-
 app/controllers/admin/domain_allows_controller.rb  | 40 ++++++++++++++++++++++
 app/controllers/admin/instances_controller.rb      | 28 +++++++++++++--
 app/controllers/api/base_controller.rb             |  9 +++++
 app/controllers/api/v1/accounts_controller.rb      |  2 ++
 app/controllers/api/v1/apps_controller.rb          |  2 ++
 .../api/v1/instances/activity_controller.rb        |  3 +-
 .../api/v1/instances/peers_controller.rb           |  3 +-
 app/controllers/api/v1/instances_controller.rb     |  1 +
 app/controllers/application_controller.rb          |  4 ++-
 app/controllers/concerns/account_owned_concern.rb  |  1 +
 app/controllers/directories_controller.rb          |  5 +--
 app/controllers/home_controller.rb                 |  2 +-
 app/controllers/media_controller.rb                |  1 +
 app/controllers/media_proxy_controller.rb          |  2 ++
 app/controllers/public_timelines_controller.rb     |  5 +--
 app/controllers/remote_interaction_controller.rb   |  1 +
 app/controllers/tags_controller.rb                 |  1 +
 app/helpers/domain_control_helper.rb               | 10 +++++-
 app/models/domain_allow.rb                         | 33 ++++++++++++++++++
 app/models/instance.rb                             |  3 +-
 app/models/instance_filter.rb                      |  4 +++
 app/policies/domain_allow_policy.rb                | 11 ++++++
 app/services/concerns/payloadable.rb               |  2 +-
 app/services/unallow_domain_service.rb             | 11 ++++++
 app/views/admin/domain_allows/new.html.haml        | 14 ++++++++
 app/views/admin/instances/index.html.haml          | 35 ++++++++++++-------
 app/views/admin/instances/show.html.haml           |  4 ++-
 app/views/admin/settings/edit.html.haml            | 28 ++++++++-------
 app/views/auth/registrations/new.html.haml         |  2 +-
 app/views/layouts/public.html.haml                 |  9 +++--
 config/initializers/2_whitelist_mode.rb            |  5 +++
 config/locales/en.yml                              |  7 ++++
 config/locales/simple_form.en.yml                  |  2 ++
 config/navigation.rb                               |  2 +-
 config/routes.rb                                   |  1 +
 db/migrate/20190705002136_create_domain_allows.rb  |  9 +++++
 db/schema.rb                                       |  9 ++++-
 lib/mastodon/domains_cli.rb                        | 22 ++++++++++--
 spec/fabricators/domain_allow_fabricator.rb        |  3 ++
 spec/models/domain_allow_spec.rb                   |  5 +++
 streaming/index.js                                 |  5 +--
 44 files changed, 302 insertions(+), 53 deletions(-)
 create mode 100644 app/controllers/admin/domain_allows_controller.rb
 create mode 100644 app/models/domain_allow.rb
 create mode 100644 app/policies/domain_allow_policy.rb
 create mode 100644 app/services/unallow_domain_service.rb
 create mode 100644 app/views/admin/domain_allows/new.html.haml
 create mode 100644 config/initializers/2_whitelist_mode.rb
 create mode 100644 db/migrate/20190705002136_create_domain_allows.rb
 create mode 100644 spec/fabricators/domain_allow_fabricator.rb
 create mode 100644 spec/models/domain_allow_spec.rb

(limited to 'app/models')

diff --git a/app/controllers/about_controller.rb b/app/controllers/about_controller.rb
index 31cf17710..d276e8fe5 100644
--- a/app/controllers/about_controller.rb
+++ b/app/controllers/about_controller.rb
@@ -3,6 +3,7 @@
 class AboutController < ApplicationController
   layout 'public'
 
+  before_action :require_open_federation!, only: [:show, :more]
   before_action :set_body_classes, only: :show
   before_action :set_instance_presenter
   before_action :set_expires_in
@@ -19,6 +20,10 @@ class AboutController < ApplicationController
 
   private
 
+  def require_open_federation!
+    not_found if whitelist_mode?
+  end
+
   def new_user
     User.new.tap do |user|
       user.build_account
diff --git a/app/controllers/activitypub/base_controller.rb b/app/controllers/activitypub/base_controller.rb
index a3b5c4dfa..0c2591e97 100644
--- a/app/controllers/activitypub/base_controller.rb
+++ b/app/controllers/activitypub/base_controller.rb
@@ -1,6 +1,8 @@
 # frozen_string_literal: true
 
 class ActivityPub::BaseController < Api::BaseController
+  skip_before_action :require_authenticated_user!
+
   private
 
   def set_cache_headers
diff --git a/app/controllers/activitypub/inboxes_controller.rb b/app/controllers/activitypub/inboxes_controller.rb
index 7cfd9a25e..bcfc1e6d4 100644
--- a/app/controllers/activitypub/inboxes_controller.rb
+++ b/app/controllers/activitypub/inboxes_controller.rb
@@ -1,6 +1,6 @@
 # frozen_string_literal: true
 
-class ActivityPub::InboxesController < Api::BaseController
+class ActivityPub::InboxesController < ActivityPub::BaseController
   include SignatureVerification
   include JsonLdHelper
   include AccountOwnedConcern
diff --git a/app/controllers/admin/domain_allows_controller.rb b/app/controllers/admin/domain_allows_controller.rb
new file mode 100644
index 000000000..31be1978b
--- /dev/null
+++ b/app/controllers/admin/domain_allows_controller.rb
@@ -0,0 +1,40 @@
+# frozen_string_literal: true
+
+class Admin::DomainAllowsController < Admin::BaseController
+  before_action :set_domain_allow, only: [:destroy]
+
+  def new
+    authorize :domain_allow, :create?
+
+    @domain_allow = DomainAllow.new(domain: params[:_domain])
+  end
+
+  def create
+    authorize :domain_allow, :create?
+
+    @domain_allow = DomainAllow.new(resource_params)
+
+    if @domain_allow.save
+      log_action :create, @domain_allow
+      redirect_to admin_instances_path, notice: I18n.t('admin.domain_allows.created_msg')
+    else
+      render :new
+    end
+  end
+
+  def destroy
+    authorize @domain_allow, :destroy?
+    UnallowDomainService.new.call(@domain_allow)
+    redirect_to admin_instances_path, notice: I18n.t('admin.domain_allows.destroyed_msg')
+  end
+
+  private
+
+  def set_domain_allow
+    @domain_allow = DomainAllow.find(params[:id])
+  end
+
+  def resource_params
+    params.require(:domain_allow).permit(:domain)
+  end
+end
diff --git a/app/controllers/admin/instances_controller.rb b/app/controllers/admin/instances_controller.rb
index 7888e844f..d4f201807 100644
--- a/app/controllers/admin/instances_controller.rb
+++ b/app/controllers/admin/instances_controller.rb
@@ -2,6 +2,10 @@
 
 module Admin
   class InstancesController < BaseController
+    before_action :set_domain_block, only: :show
+    before_action :set_domain_allow, only: :show
+    before_action :set_instance, only: :show
+
     def index
       authorize :instance, :index?
 
@@ -11,20 +15,38 @@ module Admin
     def show
       authorize :instance, :show?
 
-      @instance        = Instance.new(Account.by_domain_accounts.find_by(domain: params[:id]) || DomainBlock.find_by!(domain: params[:id]))
       @following_count = Follow.where(account: Account.where(domain: params[:id])).count
       @followers_count = Follow.where(target_account: Account.where(domain: params[:id])).count
       @reports_count   = Report.where(target_account: Account.where(domain: params[:id])).count
       @blocks_count    = Block.where(target_account: Account.where(domain: params[:id])).count
       @available       = DeliveryFailureTracker.available?(Account.select(:shared_inbox_url).where(domain: params[:id]).first&.shared_inbox_url)
       @media_storage   = MediaAttachment.where(account: Account.where(domain: params[:id])).sum(:file_file_size)
-      @domain_block    = DomainBlock.rule_for(params[:id])
     end
 
     private
 
+    def set_domain_block
+      @domain_block = DomainBlock.rule_for(params[:id])
+    end
+
+    def set_domain_allow
+      @domain_allow = DomainAllow.rule_for(params[:id])
+    end
+
+    def set_instance
+      resource   = Account.by_domain_accounts.find_by(domain: params[:id])
+      resource ||= @domain_block
+      resource ||= @domain_allow
+
+      if resource
+        @instance = Instance.new(resource)
+      else
+        not_found
+      end
+    end
+
     def filtered_instances
-      InstanceFilter.new(filter_params).results
+      InstanceFilter.new(whitelist_mode? ? { allowed: true } : filter_params).results
     end
 
     def paginated_instances
diff --git a/app/controllers/api/base_controller.rb b/app/controllers/api/base_controller.rb
index 6f33a1ea9..109e38ffa 100644
--- a/app/controllers/api/base_controller.rb
+++ b/app/controllers/api/base_controller.rb
@@ -9,6 +9,7 @@ class Api::BaseController < ApplicationController
   skip_before_action :store_current_location
   skip_before_action :require_functional!
 
+  before_action :require_authenticated_user!, if: :disallow_unauthenticated_api_access?
   before_action :set_cache_headers
 
   protect_from_forgery with: :null_session
@@ -69,6 +70,10 @@ class Api::BaseController < ApplicationController
     nil
   end
 
+  def require_authenticated_user!
+    render json: { error: 'This API requires an authenticated user' }, status: 401 unless current_user
+  end
+
   def require_user!
     if !current_user
       render json: { error: 'This method requires an authenticated user' }, status: 422
@@ -94,4 +99,8 @@ class Api::BaseController < ApplicationController
   def set_cache_headers
     response.headers['Cache-Control'] = 'no-cache, no-store, max-age=0, must-revalidate'
   end
+
+  def disallow_unauthenticated_api_access?
+    authorized_fetch_mode?
+  end
 end
diff --git a/app/controllers/api/v1/accounts_controller.rb b/app/controllers/api/v1/accounts_controller.rb
index b0c62778e..b306e8e8c 100644
--- a/app/controllers/api/v1/accounts_controller.rb
+++ b/app/controllers/api/v1/accounts_controller.rb
@@ -12,6 +12,8 @@ class Api::V1::AccountsController < Api::BaseController
   before_action :check_account_suspension, only: [:show]
   before_action :check_enabled_registrations, only: [:create]
 
+  skip_before_action :require_authenticated_user!, only: :create
+
   respond_to :json
 
   def show
diff --git a/app/controllers/api/v1/apps_controller.rb b/app/controllers/api/v1/apps_controller.rb
index e9f7a7291..97177547a 100644
--- a/app/controllers/api/v1/apps_controller.rb
+++ b/app/controllers/api/v1/apps_controller.rb
@@ -1,6 +1,8 @@
 # frozen_string_literal: true
 
 class Api::V1::AppsController < Api::BaseController
+  skip_before_action :require_authenticated_user!
+
   def create
     @app = Doorkeeper::Application.create!(application_options)
     render json: @app, serializer: REST::ApplicationSerializer
diff --git a/app/controllers/api/v1/instances/activity_controller.rb b/app/controllers/api/v1/instances/activity_controller.rb
index d0080c5c2..4fb5a69d8 100644
--- a/app/controllers/api/v1/instances/activity_controller.rb
+++ b/app/controllers/api/v1/instances/activity_controller.rb
@@ -2,6 +2,7 @@
 
 class Api::V1::Instances::ActivityController < Api::BaseController
   before_action :require_enabled_api!
+
   skip_before_action :set_cache_headers
 
   respond_to :json
@@ -33,6 +34,6 @@ class Api::V1::Instances::ActivityController < Api::BaseController
   end
 
   def require_enabled_api!
-    head 404 unless Setting.activity_api_enabled
+    head 404 unless Setting.activity_api_enabled && !whitelist_mode?
   end
 end
diff --git a/app/controllers/api/v1/instances/peers_controller.rb b/app/controllers/api/v1/instances/peers_controller.rb
index 450e6502f..75c3cb4ba 100644
--- a/app/controllers/api/v1/instances/peers_controller.rb
+++ b/app/controllers/api/v1/instances/peers_controller.rb
@@ -2,6 +2,7 @@
 
 class Api::V1::Instances::PeersController < Api::BaseController
   before_action :require_enabled_api!
+
   skip_before_action :set_cache_headers
 
   respond_to :json
@@ -14,6 +15,6 @@ class Api::V1::Instances::PeersController < Api::BaseController
   private
 
   def require_enabled_api!
-    head 404 unless Setting.peers_api_enabled
+    head 404 unless Setting.peers_api_enabled && !whitelist_mode?
   end
 end
diff --git a/app/controllers/api/v1/instances_controller.rb b/app/controllers/api/v1/instances_controller.rb
index 93e4f0003..8d8231423 100644
--- a/app/controllers/api/v1/instances_controller.rb
+++ b/app/controllers/api/v1/instances_controller.rb
@@ -2,6 +2,7 @@
 
 class Api::V1::InstancesController < Api::BaseController
   respond_to :json
+
   skip_before_action :set_cache_headers
 
   def show
diff --git a/app/controllers/application_controller.rb b/app/controllers/application_controller.rb
index 41ce1a0ca..0d3913ee0 100644
--- a/app/controllers/application_controller.rb
+++ b/app/controllers/application_controller.rb
@@ -11,12 +11,14 @@ class ApplicationController < ActionController::Base
   include UserTrackingConcern
   include SessionTrackingConcern
   include CacheConcern
+  include DomainControlHelper
 
   helper_method :current_account
   helper_method :current_session
   helper_method :current_theme
   helper_method :single_user_mode?
   helper_method :use_seamless_external_login?
+  helper_method :whitelist_mode?
 
   rescue_from ActionController::RoutingError, with: :not_found
   rescue_from ActiveRecord::RecordNotFound, with: :not_found
@@ -38,7 +40,7 @@ class ApplicationController < ActionController::Base
   end
 
   def authorized_fetch_mode?
-    ENV['AUTHORIZED_FETCH'] == 'true'
+    ENV['AUTHORIZED_FETCH'] == 'true' || Rails.configuration.x.whitelist_mode
   end
 
   def public_fetch_mode?
diff --git a/app/controllers/concerns/account_owned_concern.rb b/app/controllers/concerns/account_owned_concern.rb
index 99c240fe9..460f71f65 100644
--- a/app/controllers/concerns/account_owned_concern.rb
+++ b/app/controllers/concerns/account_owned_concern.rb
@@ -4,6 +4,7 @@ module AccountOwnedConcern
   extend ActiveSupport::Concern
 
   included do
+    before_action :authenticate_user!, if: -> { whitelist_mode? && request.format != :json }
     before_action :set_account, if: :account_required?
     before_action :check_account_approval, if: :account_required?
     before_action :check_account_suspension, if: :account_required?
diff --git a/app/controllers/directories_controller.rb b/app/controllers/directories_controller.rb
index 594907674..d2ef76f06 100644
--- a/app/controllers/directories_controller.rb
+++ b/app/controllers/directories_controller.rb
@@ -3,7 +3,8 @@
 class DirectoriesController < ApplicationController
   layout 'public'
 
-  before_action :check_enabled
+  before_action :authenticate_user!, if: :whitelist_mode?
+  before_action :require_enabled!
   before_action :set_instance_presenter
   before_action :set_tag, only: :show
   before_action :set_tags
@@ -19,7 +20,7 @@ class DirectoriesController < ApplicationController
 
   private
 
-  def check_enabled
+  def require_enabled!
     return not_found unless Setting.profile_directory
   end
 
diff --git a/app/controllers/home_controller.rb b/app/controllers/home_controller.rb
index 42493cd78..22d507e77 100644
--- a/app/controllers/home_controller.rb
+++ b/app/controllers/home_controller.rb
@@ -55,7 +55,7 @@ class HomeController < ApplicationController
   end
 
   def default_redirect_path
-    if request.path.start_with?('/web')
+    if request.path.start_with?('/web') || whitelist_mode?
       new_user_session_path
     elsif single_user_mode?
       short_account_path(Account.local.without_suspended.where('id > 0').first)
diff --git a/app/controllers/media_controller.rb b/app/controllers/media_controller.rb
index b3b7519a1..1f693de32 100644
--- a/app/controllers/media_controller.rb
+++ b/app/controllers/media_controller.rb
@@ -5,6 +5,7 @@ class MediaController < ApplicationController
 
   skip_before_action :store_current_location
 
+  before_action :authenticate_user!, if: :whitelist_mode?
   before_action :set_media_attachment
   before_action :verify_permitted_status!
   before_action :check_playable, only: :player
diff --git a/app/controllers/media_proxy_controller.rb b/app/controllers/media_proxy_controller.rb
index 8fc18dd06..8da6c6fe0 100644
--- a/app/controllers/media_proxy_controller.rb
+++ b/app/controllers/media_proxy_controller.rb
@@ -5,6 +5,8 @@ class MediaProxyController < ApplicationController
 
   skip_before_action :store_current_location
 
+  before_action :authenticate_user!, if: :whitelist_mode?
+
   def show
     RedisLock.acquire(lock_options) do |lock|
       if lock.acquired?
diff --git a/app/controllers/public_timelines_controller.rb b/app/controllers/public_timelines_controller.rb
index 23506b990..324bdc508 100644
--- a/app/controllers/public_timelines_controller.rb
+++ b/app/controllers/public_timelines_controller.rb
@@ -3,7 +3,8 @@
 class PublicTimelinesController < ApplicationController
   layout 'public'
 
-  before_action :check_enabled
+  before_action :authenticate_user!, if: :whitelist_mode?
+  before_action :require_enabled!
   before_action :set_body_classes
   before_action :set_instance_presenter
 
@@ -16,7 +17,7 @@ class PublicTimelinesController < ApplicationController
 
   private
 
-  def check_enabled
+  def require_enabled!
     not_found unless Setting.timeline_preview
   end
 
diff --git a/app/controllers/remote_interaction_controller.rb b/app/controllers/remote_interaction_controller.rb
index cc6993c52..fa742fb0a 100644
--- a/app/controllers/remote_interaction_controller.rb
+++ b/app/controllers/remote_interaction_controller.rb
@@ -5,6 +5,7 @@ class RemoteInteractionController < ApplicationController
 
   layout 'modal'
 
+  before_action :authenticate_user!, if: :whitelist_mode?
   before_action :set_interaction_type
   before_action :set_status
   before_action :set_body_classes
diff --git a/app/controllers/tags_controller.rb b/app/controllers/tags_controller.rb
index d08e5a61a..3cd2d9e20 100644
--- a/app/controllers/tags_controller.rb
+++ b/app/controllers/tags_controller.rb
@@ -8,6 +8,7 @@ class TagsController < ApplicationController
   layout 'public'
 
   before_action :require_signature!, if: -> { request.format == :json && authorized_fetch_mode? }
+  before_action :authenticate_user!, if: :whitelist_mode?
   before_action :set_tag
   before_action :set_body_classes
   before_action :set_instance_presenter
diff --git a/app/helpers/domain_control_helper.rb b/app/helpers/domain_control_helper.rb
index efd328f81..067b2c2cd 100644
--- a/app/helpers/domain_control_helper.rb
+++ b/app/helpers/domain_control_helper.rb
@@ -12,6 +12,14 @@ module DomainControlHelper
       end
     end
 
-    DomainBlock.blocked?(domain)
+    if whitelist_mode?
+      !DomainAllow.allowed?(domain)
+    else
+      DomainBlock.blocked?(domain)
+    end
+  end
+
+  def whitelist_mode?
+    Rails.configuration.x.whitelist_mode
   end
 end
diff --git a/app/models/domain_allow.rb b/app/models/domain_allow.rb
new file mode 100644
index 000000000..85018b636
--- /dev/null
+++ b/app/models/domain_allow.rb
@@ -0,0 +1,33 @@
+# frozen_string_literal: true
+
+# == Schema Information
+#
+# Table name: domain_allows
+#
+#  id         :bigint(8)        not null, primary key
+#  domain     :string           default(""), not null
+#  created_at :datetime         not null
+#  updated_at :datetime         not null
+#
+
+class DomainAllow < ApplicationRecord
+  include DomainNormalizable
+
+  validates :domain, presence: true, uniqueness: true
+
+  scope :matches_domain, ->(value) { where(arel_table[:domain].matches("%#{value}%")) }
+
+  class << self
+    def allowed?(domain)
+      !rule_for(domain).nil?
+    end
+
+    def rule_for(domain)
+      return if domain.blank?
+
+      uri = Addressable::URI.new.tap { |u| u.host = domain.gsub(/[\/]/, '') }
+
+      find_by(domain: uri.normalized_host)
+    end
+  end
+end
diff --git a/app/models/instance.rb b/app/models/instance.rb
index 797a191e0..3c740f8a2 100644
--- a/app/models/instance.rb
+++ b/app/models/instance.rb
@@ -7,8 +7,9 @@ class Instance
 
   def initialize(resource)
     @domain         = resource.domain
-    @accounts_count = resource.is_a?(DomainBlock) ? nil : resource.accounts_count
+    @accounts_count = resource.respond_to?(:accounts_count) ? resource.accounts_count : nil
     @domain_block   = resource.is_a?(DomainBlock) ? resource : DomainBlock.rule_for(domain)
+    @domain_allow   = resource.is_a?(DomainAllow) ? resource : DomainAllow.rule_for(domain)
   end
 
   def countable?
diff --git a/app/models/instance_filter.rb b/app/models/instance_filter.rb
index 848fff53e..8bfab826d 100644
--- a/app/models/instance_filter.rb
+++ b/app/models/instance_filter.rb
@@ -12,6 +12,10 @@ class InstanceFilter
       scope = DomainBlock
       scope = scope.matches_domain(params[:by_domain]) if params[:by_domain].present?
       scope.order(id: :desc)
+    elsif params[:allowed].present?
+      scope = DomainAllow
+      scope = scope.matches_domain(params[:by_domain]) if params[:by_domain].present?
+      scope.order(id: :desc)
     else
       scope = Account.remote
       scope = scope.matches_domain(params[:by_domain]) if params[:by_domain].present?
diff --git a/app/policies/domain_allow_policy.rb b/app/policies/domain_allow_policy.rb
new file mode 100644
index 000000000..5030453bb
--- /dev/null
+++ b/app/policies/domain_allow_policy.rb
@@ -0,0 +1,11 @@
+# frozen_string_literal: true
+
+class DomainAllowPolicy < ApplicationPolicy
+  def create?
+    admin?
+  end
+
+  def destroy?
+    admin?
+  end
+end
diff --git a/app/services/concerns/payloadable.rb b/app/services/concerns/payloadable.rb
index 953740faa..7f9f21c4b 100644
--- a/app/services/concerns/payloadable.rb
+++ b/app/services/concerns/payloadable.rb
@@ -14,6 +14,6 @@ module Payloadable
   end
 
   def signing_enabled?
-    ENV['AUTHORIZED_FETCH'] != 'true'
+    ENV['AUTHORIZED_FETCH'] != 'true' && !Rails.configuration.x.whitelist_mode
   end
 end
diff --git a/app/services/unallow_domain_service.rb b/app/services/unallow_domain_service.rb
new file mode 100644
index 000000000..d4387c1a1
--- /dev/null
+++ b/app/services/unallow_domain_service.rb
@@ -0,0 +1,11 @@
+# frozen_string_literal: true
+
+class UnallowDomainService < BaseService
+  def call(domain_allow)
+    Account.where(domain: domain_allow.domain).find_each do |account|
+      SuspendAccountService.new.call(account, destroy: true)
+    end
+
+    domain_allow.destroy
+  end
+end
diff --git a/app/views/admin/domain_allows/new.html.haml b/app/views/admin/domain_allows/new.html.haml
new file mode 100644
index 000000000..52599857a
--- /dev/null
+++ b/app/views/admin/domain_allows/new.html.haml
@@ -0,0 +1,14 @@
+- content_for :header_tags do
+  = javascript_pack_tag 'admin', integrity: true, async: true, crossorigin: 'anonymous'
+
+- content_for :page_title do
+  = t('admin.domain_allows.add_new')
+
+= simple_form_for @domain_allow, url: admin_domain_allows_path do |f|
+  = render 'shared/error_messages', object: @domain_allow
+
+  .fields-group
+    = f.input :domain, wrapper: :with_label, label: t('admin.domain_blocks.domain'), required: true
+
+  .actions
+    = f.button :button, t('admin.domain_allows.add_new'), type: :submit
diff --git a/app/views/admin/instances/index.html.haml b/app/views/admin/instances/index.html.haml
index 61e578409..982dc5035 100644
--- a/app/views/admin/instances/index.html.haml
+++ b/app/views/admin/instances/index.html.haml
@@ -6,24 +6,30 @@
     %strong= t('admin.instances.moderation.title')
     %ul
       %li= filter_link_to t('admin.instances.moderation.all'), limited: nil
-      %li= filter_link_to t('admin.instances.moderation.limited'), limited: '1'
+
+      - unless whitelist_mode?
+        %li= filter_link_to t('admin.instances.moderation.limited'), limited: '1'
 
   %div{ style: 'flex: 1 1 auto; text-align: right' }
-    = link_to t('admin.domain_blocks.add_new'), new_admin_domain_block_path, class: 'button'
+    - if whitelist_mode?
+      = link_to t('admin.domain_allows.add_new'), new_admin_domain_allow_path, class: 'button'
+    - else
+      = link_to t('admin.domain_blocks.add_new'), new_admin_domain_block_path, class: 'button'
 
-= form_tag admin_instances_url, method: 'GET', class: 'simple_form' do
-  .fields-group
-    - Admin::FilterHelper::INSTANCES_FILTERS.each do |key|
-      - if params[key].present?
-        = hidden_field_tag key, params[key]
+- unless whitelist_mode?
+  = form_tag admin_instances_url, method: 'GET', class: 'simple_form' do
+    .fields-group
+      - Admin::FilterHelper::INSTANCES_FILTERS.each do |key|
+        - if params[key].present?
+          = hidden_field_tag key, params[key]
 
-    - %i(by_domain).each do |key|
-      .input.string.optional
-        = text_field_tag key, params[key], class: 'string optional', placeholder: I18n.t("admin.instances.#{key}")
+      - %i(by_domain).each do |key|
+        .input.string.optional
+          = text_field_tag key, params[key], class: 'string optional', placeholder: I18n.t("admin.instances.#{key}")
 
-    .actions
-      %button= t('admin.accounts.search')
-      = link_to t('admin.accounts.reset'), admin_instances_path, class: 'button negative'
+      .actions
+        %button= t('admin.accounts.search')
+        = link_to t('admin.accounts.reset'), admin_instances_path, class: 'button negative'
 
 %hr.spacer/
 
@@ -47,8 +53,11 @@
               - unless first_item
                 &bull;
               = t('admin.domain_blocks.rejecting_reports')
+          - elsif whitelist_mode?
+            = t('admin.accounts.whitelisted')
           - else
             = t('admin.accounts.no_limits_imposed')
       - if instance.countable?
         .trends__item__current{ title: t('admin.instances.known_accounts', count: instance.accounts_count) }= number_to_human instance.accounts_count, strip_insignificant_zeros: true
+
 = paginate paginated_instances
diff --git a/app/views/admin/instances/show.html.haml b/app/views/admin/instances/show.html.haml
index c7992a490..fbb49ba02 100644
--- a/app/views/admin/instances/show.html.haml
+++ b/app/views/admin/instances/show.html.haml
@@ -38,7 +38,9 @@
     = link_to t('admin.accounts.title'), admin_accounts_path(remote: '1', by_domain: @instance.domain), class: 'button'
 
   %div{ style: 'float: right' }
-    - if @domain_block
+    - if @domain_allow
+      = link_to t('admin.domain_allows.undo'), admin_domain_allow_path(@domain_allow), class: 'button button--destructive', data: { confirm: t('admin.accounts.are_you_sure'), method: :delete }
+    - elsif @domain_block
       = link_to t('admin.domain_blocks.undo'), admin_domain_block_path(@domain_block), class: 'button'
     - else
       = link_to t('admin.domain_blocks.add_new'), new_admin_domain_block_path(_domain: @instance.domain), class: 'button'
diff --git a/app/views/admin/settings/edit.html.haml b/app/views/admin/settings/edit.html.haml
index b3bf3849c..1e2ed3f77 100644
--- a/app/views/admin/settings/edit.html.haml
+++ b/app/views/admin/settings/edit.html.haml
@@ -42,11 +42,12 @@
 
   %hr.spacer/
 
-  .fields-group
-    = f.input :timeline_preview, as: :boolean, wrapper: :with_label, label: t('admin.settings.timeline_preview.title'), hint: t('admin.settings.timeline_preview.desc_html')
+  - unless whitelist_mode?
+    .fields-group
+      = f.input :timeline_preview, as: :boolean, wrapper: :with_label, label: t('admin.settings.timeline_preview.title'), hint: t('admin.settings.timeline_preview.desc_html')
 
-  .fields-group
-    = f.input :show_known_fediverse_at_about_page, as: :boolean, wrapper: :with_label, label: t('admin.settings.show_known_fediverse_at_about_page.title'), hint: t('admin.settings.show_known_fediverse_at_about_page.desc_html')
+    .fields-group
+      = f.input :show_known_fediverse_at_about_page, as: :boolean, wrapper: :with_label, label: t('admin.settings.show_known_fediverse_at_about_page.title'), hint: t('admin.settings.show_known_fediverse_at_about_page.desc_html')
 
   .fields-group
     = f.input :show_staff_badge, as: :boolean, wrapper: :with_label, label: t('admin.settings.show_staff_badge.title'), hint: t('admin.settings.show_staff_badge.desc_html')
@@ -54,17 +55,18 @@
   .fields-group
     = f.input :open_deletion, as: :boolean, wrapper: :with_label, label: t('admin.settings.registrations.deletion.title'), hint: t('admin.settings.registrations.deletion.desc_html')
 
-  .fields-group
-    = f.input :activity_api_enabled, as: :boolean, wrapper: :with_label, label: t('admin.settings.activity_api_enabled.title'), hint: t('admin.settings.activity_api_enabled.desc_html')
+  - unless whitelist_mode?
+    .fields-group
+      = f.input :activity_api_enabled, as: :boolean, wrapper: :with_label, label: t('admin.settings.activity_api_enabled.title'), hint: t('admin.settings.activity_api_enabled.desc_html')
 
-  .fields-group
-    = f.input :peers_api_enabled, as: :boolean, wrapper: :with_label, label: t('admin.settings.peers_api_enabled.title'), hint: t('admin.settings.peers_api_enabled.desc_html')
+    .fields-group
+      = f.input :peers_api_enabled, as: :boolean, wrapper: :with_label, label: t('admin.settings.peers_api_enabled.title'), hint: t('admin.settings.peers_api_enabled.desc_html')
 
-  .fields-group
-    = f.input :preview_sensitive_media, as: :boolean, wrapper: :with_label, label: t('admin.settings.preview_sensitive_media.title'), hint: t('admin.settings.preview_sensitive_media.desc_html')
+    .fields-group
+      = f.input :preview_sensitive_media, as: :boolean, wrapper: :with_label, label: t('admin.settings.preview_sensitive_media.title'), hint: t('admin.settings.preview_sensitive_media.desc_html')
 
-  .fields-group
-    = f.input :profile_directory, as: :boolean, wrapper: :with_label, label: t('admin.settings.profile_directory.title'), hint: t('admin.settings.profile_directory.desc_html')
+    .fields-group
+      = f.input :profile_directory, as: :boolean, wrapper: :with_label, label: t('admin.settings.profile_directory.title'), hint: t('admin.settings.profile_directory.desc_html')
 
   .fields-group
     = f.input :spam_check_enabled, as: :boolean, wrapper: :with_label, label: t('admin.settings.spam_check_enabled.title'), hint: t('admin.settings.spam_check_enabled.desc_html')
@@ -76,7 +78,7 @@
 
   .fields-group
     = f.input :closed_registrations_message, as: :text, wrapper: :with_block_label, label: t('admin.settings.registrations.closed_message.title'), hint: t('admin.settings.registrations.closed_message.desc_html'), input_html: { rows: 8 }
-    = f.input :site_extended_description, wrapper: :with_block_label, as: :text, label: t('admin.settings.site_description_extended.title'), hint: t('admin.settings.site_description_extended.desc_html'), input_html: { rows: 8 }
+    = f.input :site_extended_description, wrapper: :with_block_label, as: :text, label: t('admin.settings.site_description_extended.title'), hint: t('admin.settings.site_description_extended.desc_html'), input_html: { rows: 8 } unless whitelist_mode?
     = f.input :site_terms, wrapper: :with_block_label, as: :text, label: t('admin.settings.site_terms.title'), hint: t('admin.settings.site_terms.desc_html'), input_html: { rows: 8 }
     = f.input :custom_css, wrapper: :with_block_label, as: :text, input_html: { rows: 8 }, label: t('admin.settings.custom_css.title'), hint: t('admin.settings.custom_css.desc_html')
 
diff --git a/app/views/auth/registrations/new.html.haml b/app/views/auth/registrations/new.html.haml
index b4a7cced5..83384d737 100644
--- a/app/views/auth/registrations/new.html.haml
+++ b/app/views/auth/registrations/new.html.haml
@@ -33,7 +33,7 @@
   = f.input :invite_code, as: :hidden
 
   .fields-group
-    = f.input :agreement, as: :boolean, wrapper: :with_label, label: t('auth.checkbox_agreement_html', rules_path: about_more_path, terms_path: terms_path)
+    = f.input :agreement, as: :boolean, wrapper: :with_label, label: whitelist_mode? ? t('auth.checkbox_agreement_without_rules_html', terms_path: terms_path) : t('auth.checkbox_agreement_html', rules_path: about_more_path, terms_path: terms_path)
 
   .actions
     = f.button :button, @invite.present? ? t('auth.register') : sign_up_message, type: :submit
diff --git a/app/views/layouts/public.html.haml b/app/views/layouts/public.html.haml
index 2929ac599..69738a2f7 100644
--- a/app/views/layouts/public.html.haml
+++ b/app/views/layouts/public.html.haml
@@ -10,10 +10,13 @@
             = link_to root_url, class: 'brand' do
               = svg_logo_full
 
-            = link_to t('directories.directory'), explore_path, class: 'nav-link optional' if Setting.profile_directory
-            = link_to t('about.about_this'), about_more_path, class: 'nav-link optional'
-            = link_to t('about.apps'), 'https://joinmastodon.org/apps', class: 'nav-link optional'
+            - unless whitelist_mode?
+              = link_to t('directories.directory'), explore_path, class: 'nav-link optional' if Setting.profile_directory
+              = link_to t('about.about_this'), about_more_path, class: 'nav-link optional'
+              = link_to t('about.apps'), 'https://joinmastodon.org/apps', class: 'nav-link optional'
+
           .nav-center
+
           .nav-right
             - if user_signed_in?
               = link_to t('settings.back'), root_url, class: 'nav-link nav-button webapp-btn'
diff --git a/config/initializers/2_whitelist_mode.rb b/config/initializers/2_whitelist_mode.rb
new file mode 100644
index 000000000..a17ad07a2
--- /dev/null
+++ b/config/initializers/2_whitelist_mode.rb
@@ -0,0 +1,5 @@
+# frozen_string_literal: true
+
+Rails.application.configure do
+  config.x.whitelist_mode = ENV['WHITELIST_MODE'] == 'true'
+end
diff --git a/config/locales/en.yml b/config/locales/en.yml
index 9e1be87be..6c1a34300 100644
--- a/config/locales/en.yml
+++ b/config/locales/en.yml
@@ -186,6 +186,7 @@ en:
       username: Username
       warn: Warn
       web: Web
+      whitelisted: Whitelisted
     action_logs:
       actions:
         assigned_to_self_report: "%{name} assigned report %{target} to themselves"
@@ -269,6 +270,11 @@ en:
       week_interactions: interactions this week
       week_users_active: active this week
       week_users_new: users this week
+    domain_allows:
+      add_new: Whitelist domain
+      created_msg: Domain has been successfully whitelisted
+      destroyed_msg: Domain has been removed from the whitelist
+      undo: Remove from whitelist
     domain_blocks:
       add_new: Add new domain block
       created_msg: Domain block is now being processed
@@ -524,6 +530,7 @@ en:
     apply_for_account: Request an invite
     change_password: Password
     checkbox_agreement_html: I agree to the <a href="%{rules_path}" target="_blank">server rules</a> and <a href="%{terms_path}" target="_blank">terms of service</a>
+    checkbox_agreement_without_rules_html: I agree to the <a href="%{terms_path}" target="_blank">terms of service</a>
     delete_account: Delete account
     delete_account_html: If you wish to delete your account, you can <a href="%{path}">proceed here</a>. You will be asked for confirmation.
     didnt_get_confirmation: Didn't receive confirmation instructions?
diff --git a/config/locales/simple_form.en.yml b/config/locales/simple_form.en.yml
index 12a7ec2b3..10b30e627 100644
--- a/config/locales/simple_form.en.yml
+++ b/config/locales/simple_form.en.yml
@@ -38,6 +38,8 @@ en:
         setting_use_pending_items: Hide timeline updates behind a click instead of automatically scrolling the feed
         username: Your username will be unique on %{domain}
         whole_word: When the keyword or phrase is alphanumeric only, it will only be applied if it matches the whole word
+      domain_allow:
+        domain: This domain will be able to fetch data from this server and incoming data from it will be processed and stored
       featured_tag:
         name: 'You might want to use one of these:'
       imports:
diff --git a/config/navigation.rb b/config/navigation.rb
index 5ab2e4399..9b46da603 100644
--- a/config/navigation.rb
+++ b/config/navigation.rb
@@ -39,7 +39,7 @@ SimpleNavigation::Configuration.run do |navigation|
       s.item :accounts, safe_join([fa_icon('users fw'), t('admin.accounts.title')]), admin_accounts_url, highlights_on: %r{/admin/accounts|/admin/pending_accounts}
       s.item :invites, safe_join([fa_icon('user-plus fw'), t('admin.invites.title')]), admin_invites_path
       s.item :tags, safe_join([fa_icon('tag fw'), t('admin.tags.title')]), admin_tags_path
-      s.item :instances, safe_join([fa_icon('cloud fw'), t('admin.instances.title')]), admin_instances_url(limited: '1'), highlights_on: %r{/admin/instances|/admin/domain_blocks}, if: -> { current_user.admin? }
+      s.item :instances, safe_join([fa_icon('cloud fw'), t('admin.instances.title')]), admin_instances_url(limited: whitelist_mode? ? nil : '1'), highlights_on: %r{/admin/instances|/admin/domain_blocks|/admin/domain_allows}, if: -> { current_user.admin? }
       s.item :email_domain_blocks, safe_join([fa_icon('envelope fw'), t('admin.email_domain_blocks.title')]), admin_email_domain_blocks_url, highlights_on: %r{/admin/email_domain_blocks}, if: -> { current_user.admin? }
     end
 
diff --git a/config/routes.rb b/config/routes.rb
index b6c215888..04424bbbd 100644
--- a/config/routes.rb
+++ b/config/routes.rb
@@ -154,6 +154,7 @@ Rails.application.routes.draw do
   namespace :admin do
     get '/dashboard', to: 'dashboard#index'
 
+    resources :domain_allows, only: [:new, :create, :show, :destroy]
     resources :domain_blocks, only: [:new, :create, :show, :destroy]
     resources :email_domain_blocks, only: [:index, :new, :create, :destroy]
     resources :action_logs, only: [:index]
diff --git a/db/migrate/20190705002136_create_domain_allows.rb b/db/migrate/20190705002136_create_domain_allows.rb
new file mode 100644
index 000000000..83b0728d9
--- /dev/null
+++ b/db/migrate/20190705002136_create_domain_allows.rb
@@ -0,0 +1,9 @@
+class CreateDomainAllows < ActiveRecord::Migration[5.2]
+  def change
+    create_table :domain_allows do |t|
+      t.string :domain, default: '', null: false, index: { unique: true }
+
+      t.timestamps
+    end
+  end
+end
diff --git a/db/schema.rb b/db/schema.rb
index 1847305c7..2d83d8b76 100644
--- a/db/schema.rb
+++ b/db/schema.rb
@@ -10,7 +10,7 @@
 #
 # It's strongly recommended that you check this file into your version control system.
 
-ActiveRecord::Schema.define(version: 2019_07_26_175042) do
+ActiveRecord::Schema.define(version: 2019_07_28_084117) do
 
   # These are extensions that must be enabled in order to support this database
   enable_extension "plpgsql"
@@ -245,6 +245,13 @@ ActiveRecord::Schema.define(version: 2019_07_26_175042) do
     t.index ["account_id"], name: "index_custom_filters_on_account_id"
   end
 
+  create_table "domain_allows", force: :cascade do |t|
+    t.string "domain", default: "", null: false
+    t.datetime "created_at", null: false
+    t.datetime "updated_at", null: false
+    t.index ["domain"], name: "index_domain_allows_on_domain", unique: true
+  end
+
   create_table "domain_blocks", force: :cascade do |t|
     t.string "domain", default: "", null: false
     t.datetime "created_at", null: false
diff --git a/lib/mastodon/domains_cli.rb b/lib/mastodon/domains_cli.rb
index b081581fe..f30062363 100644
--- a/lib/mastodon/domains_cli.rb
+++ b/lib/mastodon/domains_cli.rb
@@ -12,17 +12,33 @@ module Mastodon
     end
 
     option :dry_run, type: :boolean
-    desc 'purge DOMAIN', 'Remove accounts from a DOMAIN without a trace'
+    option :whitelist_mode, type: :boolean
+    desc 'purge [DOMAIN]', 'Remove accounts from a DOMAIN without a trace'
     long_desc <<-LONG_DESC
       Remove all accounts from a given DOMAIN without leaving behind any
       records. Unlike a suspension, if the DOMAIN still exists in the wild,
       it means the accounts could return if they are resolved again.
+
+      When the --whitelist-mode option is given, instead of purging accounts
+      from a single domain, all accounts from domains that are not whitelisted
+      are removed from the database.
     LONG_DESC
-    def purge(domain)
+    def purge(domain = nil)
       removed = 0
       dry_run = options[:dry_run] ? ' (DRY RUN)' : ''
 
-      Account.where(domain: domain).find_each do |account|
+      scope = begin
+        if options[:whitelist_mode]
+          Account.remote.where.not(domain: DomainAllow.pluck(:domain))
+        elsif domain.present?
+          Account.remote.where(domain: domain)
+        else
+          say('No domain given', :red)
+          exit(1)
+        end
+      end
+
+      scope.find_each do |account|
         SuspendAccountService.new.call(account, destroy: true) unless options[:dry_run]
         removed += 1
         say('.', :green, false)
diff --git a/spec/fabricators/domain_allow_fabricator.rb b/spec/fabricators/domain_allow_fabricator.rb
new file mode 100644
index 000000000..6226b1e20
--- /dev/null
+++ b/spec/fabricators/domain_allow_fabricator.rb
@@ -0,0 +1,3 @@
+Fabricator(:domain_allow) do
+  domain "MyString"
+end
diff --git a/spec/models/domain_allow_spec.rb b/spec/models/domain_allow_spec.rb
new file mode 100644
index 000000000..e65435127
--- /dev/null
+++ b/spec/models/domain_allow_spec.rb
@@ -0,0 +1,5 @@
+require 'rails_helper'
+
+RSpec.describe DomainAllow, type: :model do
+  pending "add some examples to (or delete) #{__FILE__}"
+end
diff --git a/streaming/index.js b/streaming/index.js
index 0529804b1..304e7e046 100644
--- a/streaming/index.js
+++ b/streaming/index.js
@@ -12,6 +12,7 @@ const uuid = require('uuid');
 const fs = require('fs');
 
 const env = process.env.NODE_ENV || 'development';
+const alwaysRequireAuth = process.env.WHITELIST_MODE === 'true' || process.env.AUTHORIZED_FETCH === 'true';
 
 dotenv.config({
   path: env === 'production' ? '.env.production' : '.env',
@@ -271,7 +272,7 @@ const startWorker = (workerId) => {
 
   const wsVerifyClient = (info, cb) => {
     const location = url.parse(info.req.url, true);
-    const authRequired = !PUBLIC_STREAMS.some(stream => stream === location.query.stream);
+    const authRequired = alwaysRequireAuth || !PUBLIC_STREAMS.some(stream => stream === location.query.stream);
     const allowedScopes = [];
 
     if (authRequired) {
@@ -306,7 +307,7 @@ const startWorker = (workerId) => {
       return;
     }
 
-    const authRequired = !PUBLIC_ENDPOINTS.some(endpoint => endpoint === req.path);
+    const authRequired = alwaysRequireAuth || !PUBLIC_ENDPOINTS.some(endpoint => endpoint === req.path);
     const allowedScopes = [];
 
     if (authRequired) {
-- 
cgit 


From 648cdbc04a21580a89d337edb0f45308aff1b93f Mon Sep 17 00:00:00 2001
From: Eugen Rochko <eugen@zeonfederated.com>
Date: Tue, 30 Jul 2019 13:10:40 +0200
Subject: Add hashtag score for better sorting of autosuggestions (#11427)

* Add hashtag score for better sorting of autosuggestions

* Do not use `~<~` operator with no text_pattern_ops index
---
 app/javascript/mastodon/reducers/compose.js    | 4 ++--
 app/models/tag.rb                              | 3 ++-
 app/models/trending_tags.rb                    | 7 ++++++-
 db/migrate/20190729185330_add_score_to_tags.rb | 5 +++++
 db/schema.rb                                   | 3 ++-
 5 files changed, 17 insertions(+), 5 deletions(-)
 create mode 100644 db/migrate/20190729185330_add_score_to_tags.rb

(limited to 'app/models')

diff --git a/app/javascript/mastodon/reducers/compose.js b/app/javascript/mastodon/reducers/compose.js
index e683a9c1a..7b0cdd5a5 100644
--- a/app/javascript/mastodon/reducers/compose.js
+++ b/app/javascript/mastodon/reducers/compose.js
@@ -153,9 +153,9 @@ const sortHashtagsByUse = (state, tags) => {
     if (usedA === usedB) {
       return 0;
     } else if (usedA && !usedB) {
-      return 1;
-    } else {
       return -1;
+    } else {
+      return 1;
     }
   });
 };
diff --git a/app/models/tag.rb b/app/models/tag.rb
index 46e3a3ec0..a2d6078f4 100644
--- a/app/models/tag.rb
+++ b/app/models/tag.rb
@@ -7,6 +7,7 @@
 #  name       :string           default(""), not null
 #  created_at :datetime         not null
 #  updated_at :datetime         not null
+#  score      :integer
 #
 
 class Tag < ApplicationRecord
@@ -78,7 +79,7 @@ class Tag < ApplicationRecord
       pattern = sanitize_sql_like(normalize(term.strip)) + '%'
 
       Tag.where(arel_table[:name].lower.matches(pattern.mb_chars.downcase.to_s))
-         .order(:name)
+         .order(Arel.sql('length(name) ASC, score DESC, name ASC'))
          .limit(limit)
          .offset(offset)
     end
diff --git a/app/models/trending_tags.rb b/app/models/trending_tags.rb
index 148535c21..34a4abbc5 100644
--- a/app/models/trending_tags.rb
+++ b/app/models/trending_tags.rb
@@ -48,12 +48,17 @@ class TrendingTags
         redis.zrem(key, tag_id.to_s)
       else
         score = ((observed - expected)**2) / expected
-        redis.zadd(key, score, tag_id.to_s)
+        added = redis.zadd(key, score, tag_id.to_s)
+        bump_tag_score!(tag_id) if added == 1
       end
 
       redis.expire(key, EXPIRE_TRENDS_AFTER)
     end
 
+    def bump_tag_score!(tag_id)
+      Tag.where(id: tag_id).update_all('score = COALESCE(score, 0) + 1')
+    end
+
     def disallowed_hashtags
       return @disallowed_hashtags if defined?(@disallowed_hashtags)
 
diff --git a/db/migrate/20190729185330_add_score_to_tags.rb b/db/migrate/20190729185330_add_score_to_tags.rb
new file mode 100644
index 000000000..75fee4b57
--- /dev/null
+++ b/db/migrate/20190729185330_add_score_to_tags.rb
@@ -0,0 +1,5 @@
+class AddScoreToTags < ActiveRecord::Migration[5.2]
+  def change
+    add_column :tags, :score, :int
+  end
+end
diff --git a/db/schema.rb b/db/schema.rb
index 2d83d8b76..e3af9c31a 100644
--- a/db/schema.rb
+++ b/db/schema.rb
@@ -10,7 +10,7 @@
 #
 # It's strongly recommended that you check this file into your version control system.
 
-ActiveRecord::Schema.define(version: 2019_07_28_084117) do
+ActiveRecord::Schema.define(version: 2019_07_29_185330) do
 
   # These are extensions that must be enabled in order to support this database
   enable_extension "plpgsql"
@@ -659,6 +659,7 @@ ActiveRecord::Schema.define(version: 2019_07_28_084117) do
     t.string "name", default: "", null: false
     t.datetime "created_at", null: false
     t.datetime "updated_at", null: false
+    t.integer "score"
     t.index "lower((name)::text)", name: "index_tags_on_name_lower", unique: true
   end
 
-- 
cgit 


From 92de439c04785530ce15f55cae18590136c75216 Mon Sep 17 00:00:00 2001
From: Eugen Rochko <eugen@zeonfederated.com>
Date: Tue, 30 Jul 2019 20:29:50 +0200
Subject: Change hashtag search to only return results that have trended in the
 past (#11448)

* Change hashtag search to only return results that have trended in the past

A way to eliminate typos and other one-off "junk" results

* Fix excluding exact matches that don't have a score

* Fix tests
---
 app/models/tag.rb       | 6 ++++--
 spec/models/tag_spec.rb | 4 ++--
 2 files changed, 6 insertions(+), 4 deletions(-)

(limited to 'app/models')

diff --git a/app/models/tag.rb b/app/models/tag.rb
index a2d6078f4..c7f0af86d 100644
--- a/app/models/tag.rb
+++ b/app/models/tag.rb
@@ -76,9 +76,11 @@ class Tag < ApplicationRecord
     end
 
     def search_for(term, limit = 5, offset = 0)
-      pattern = sanitize_sql_like(normalize(term.strip)) + '%'
+      normalized_term = normalize(term.strip).mb_chars.downcase.to_s
+      pattern         = sanitize_sql_like(normalized_term) + '%'
 
-      Tag.where(arel_table[:name].lower.matches(pattern.mb_chars.downcase.to_s))
+      Tag.where(arel_table[:name].lower.matches(pattern))
+         .where(arel_table[:score].gt(0).or(arel_table[:name].lower.eq(normalized_term)))
          .order(Arel.sql('length(name) ASC, score DESC, name ASC'))
          .limit(limit)
          .offset(offset)
diff --git a/spec/models/tag_spec.rb b/spec/models/tag_spec.rb
index 5f07fd618..9d700849b 100644
--- a/spec/models/tag_spec.rb
+++ b/spec/models/tag_spec.rb
@@ -136,8 +136,8 @@ RSpec.describe Tag, type: :model do
     end
 
     it 'finds the exact matching tag as the first item' do
-      similar_tag = Fabricate(:tag, name: "matchlater")
-      tag = Fabricate(:tag, name: "match")
+      similar_tag = Fabricate(:tag, name: "matchlater", score: 1)
+      tag = Fabricate(:tag, name: "match", score: 1)
 
       results = Tag.search_for("match")
 
-- 
cgit 


From 29609c09f4f6e50ebba60c8472e015c16275a4d2 Mon Sep 17 00:00:00 2001
From: Eugen Rochko <eugen@zeonfederated.com>
Date: Thu, 1 Aug 2019 19:57:09 +0200
Subject: Fix tag score not being updated when the tag is trending (#11465)

---
 app/models/trending_tags.rb | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

(limited to 'app/models')

diff --git a/app/models/trending_tags.rb b/app/models/trending_tags.rb
index 34a4abbc5..211c8f1dc 100644
--- a/app/models/trending_tags.rb
+++ b/app/models/trending_tags.rb
@@ -49,7 +49,7 @@ class TrendingTags
       else
         score = ((observed - expected)**2) / expected
         added = redis.zadd(key, score, tag_id.to_s)
-        bump_tag_score!(tag_id) if added == 1
+        bump_tag_score!(tag_id) if added
       end
 
       redis.expire(key, EXPIRE_TRENDS_AFTER)
-- 
cgit 


From 115dab78f1cc5357281dcb593f04ac8b2629cec6 Mon Sep 17 00:00:00 2001
From: Eugen Rochko <eugen@zeonfederated.com>
Date: Mon, 5 Aug 2019 19:54:29 +0200
Subject: Change admin UI for hashtags and add back whitelisted trends (#11490)

Fix #271

Add back the `GET /api/v1/trends` API with the caveat that it does
not return tags that have not been allowed to trend by the staff.

When a hashtag begins to trend (internally) and that hashtag has
not been previously reviewed by the staff, the staff is notified.

The new admin UI for hashtags allows filtering hashtags by where
they are used (e.g. in the profile directory), whether they have
been reviewed or are pending reviewal, they show by how many people
the hashtag is used in the directory, how many people used it
today, how many statuses with it have been created today, and it
allows fixing the name of the hashtag to make it more readable.

The disallowed hashtags feature has been reworked. It is now
controlled from the admin UI for hashtags instead of from
the file `config/settings.yml`
---
 app/controllers/admin/dashboard_controller.rb      |  2 +-
 app/controllers/admin/tags_controller.rb           | 36 ++++++++-----
 app/controllers/api/v1/trends_controller.rb        | 17 ++++++
 app/controllers/settings/preferences_controller.rb |  2 +-
 app/helpers/admin/filter_helper.rb                 |  5 +-
 app/mailers/admin_mailer.rb                        | 10 ++++
 app/models/application_record.rb                   | 11 ++++
 app/models/tag.rb                                  | 60 +++++++++++++++++++---
 app/models/trending_tags.rb                        | 48 ++++++++---------
 app/models/user.rb                                 |  4 ++
 app/policies/tag_policy.rb                         |  4 +-
 app/validators/disallowed_hashtags_validator.rb    | 21 +-------
 app/views/admin/dashboard/index.html.haml          |  2 +-
 app/views/admin/tags/_tag.html.haml                | 24 +++++----
 app/views/admin/tags/index.html.haml               | 26 +++++-----
 app/views/admin/tags/show.html.haml                | 16 ++++++
 app/views/admin_mailer/new_trending_tag.text.erb   |  5 ++
 .../preferences/notifications/show.html.haml       |  1 +
 config/locales/en.yml                              | 18 ++++---
 config/locales/simple_form.en.yml                  |  7 +++
 config/navigation.rb                               |  2 +-
 config/routes.rb                                   |  9 +---
 config/settings.yml                                |  1 +
 .../20190805123746_add_capabilities_to_tags.rb     |  9 ++++
 db/schema.rb                                       |  7 ++-
 spec/controllers/admin/tags_controller_spec.rb     | 56 ++------------------
 spec/policies/tag_policy_spec.rb                   |  2 +-
 .../disallowed_hashtags_validator_spec.rb          | 26 +++++-----
 28 files changed, 258 insertions(+), 173 deletions(-)
 create mode 100644 app/controllers/api/v1/trends_controller.rb
 create mode 100644 app/views/admin/tags/show.html.haml
 create mode 100644 app/views/admin_mailer/new_trending_tag.text.erb
 create mode 100644 db/migrate/20190805123746_add_capabilities_to_tags.rb

(limited to 'app/models')

diff --git a/app/controllers/admin/dashboard_controller.rb b/app/controllers/admin/dashboard_controller.rb
index e74e4755f..70afdedd7 100644
--- a/app/controllers/admin/dashboard_controller.rb
+++ b/app/controllers/admin/dashboard_controller.rb
@@ -27,7 +27,7 @@ module Admin
       @saml_enabled          = ENV['SAML_ENABLED'] == 'true'
       @pam_enabled           = ENV['PAM_ENABLED'] == 'true'
       @hidden_service        = ENV['ALLOW_ACCESS_TO_HIDDEN_SERVICE'] == 'true'
-      @trending_hashtags     = TrendingTags.get(7)
+      @trending_hashtags     = TrendingTags.get(10, filtered: false)
       @profile_directory     = Setting.profile_directory
       @timeline_preview      = Setting.timeline_preview
       @spam_check_enabled    = Setting.spam_check_enabled
diff --git a/app/controllers/admin/tags_controller.rb b/app/controllers/admin/tags_controller.rb
index e9f4f2cfa..0e9dda302 100644
--- a/app/controllers/admin/tags_controller.rb
+++ b/app/controllers/admin/tags_controller.rb
@@ -4,41 +4,49 @@ module Admin
   class TagsController < BaseController
     before_action :set_tags, only: :index
     before_action :set_tag, except: :index
-    before_action :set_filter_params
 
     def index
       authorize :tag, :index?
     end
 
-    def hide
-      authorize @tag, :hide?
-      @tag.account_tag_stat.update!(hidden: true)
-      redirect_to admin_tags_path(@filter_params)
+    def show
+      authorize @tag, :show?
     end
 
-    def unhide
-      authorize @tag, :unhide?
-      @tag.account_tag_stat.update!(hidden: false)
-      redirect_to admin_tags_path(@filter_params)
+    def update
+      authorize @tag, :update?
+
+      if @tag.update(tag_params.merge(reviewed_at: Time.now.utc))
+        redirect_to admin_tag_path(@tag.id)
+      else
+        render :show
+      end
     end
 
     private
 
     def set_tags
-      @tags = Tag.discoverable
-      @tags.merge!(Tag.hidden) if filter_params[:hidden]
+      @tags = filtered_tags.page(params[:page])
     end
 
     def set_tag
       @tag = Tag.find(params[:id])
     end
 
-    def set_filter_params
-      @filter_params = filter_params.to_hash.symbolize_keys
+    def filtered_tags
+      scope = Tag
+      scope = scope.discoverable if filter_params[:context] == 'directory'
+      scope = scope.reviewed if filter_params[:review] == 'reviewed'
+      scope = scope.pending_review if filter_params[:review] == 'pending_review'
+      scope.reorder(score: :desc)
     end
 
     def filter_params
-      params.permit(:hidden)
+      params.slice(:context, :review).permit(:context, :review)
+    end
+
+    def tag_params
+      params.require(:tag).permit(:name, :trendable, :usable, :listable)
     end
   end
 end
diff --git a/app/controllers/api/v1/trends_controller.rb b/app/controllers/api/v1/trends_controller.rb
new file mode 100644
index 000000000..bcea9857e
--- /dev/null
+++ b/app/controllers/api/v1/trends_controller.rb
@@ -0,0 +1,17 @@
+# frozen_string_literal: true
+
+class Api::V1::TrendsController < Api::BaseController
+  before_action :set_tags
+
+  respond_to :json
+
+  def index
+    render json: @tags, each_serializer: REST::TagSerializer
+  end
+
+  private
+
+  def set_tags
+    @tags = TrendingTags.get(limit_param(10))
+  end
+end
diff --git a/app/controllers/settings/preferences_controller.rb b/app/controllers/settings/preferences_controller.rb
index 742c97cdb..d548072a8 100644
--- a/app/controllers/settings/preferences_controller.rb
+++ b/app/controllers/settings/preferences_controller.rb
@@ -56,7 +56,7 @@ class Settings::PreferencesController < Settings::BaseController
       :setting_advanced_layout,
       :setting_use_blurhash,
       :setting_use_pending_items,
-      notification_emails: %i(follow follow_request reblog favourite mention digest report pending_account),
+      notification_emails: %i(follow follow_request reblog favourite mention digest report pending_account trending_tag),
       interactions: %i(must_be_follower must_be_following must_be_following_dm)
     )
   end
diff --git a/app/helpers/admin/filter_helper.rb b/app/helpers/admin/filter_helper.rb
index 0bda25974..506429e10 100644
--- a/app/helpers/admin/filter_helper.rb
+++ b/app/helpers/admin/filter_helper.rb
@@ -5,15 +5,16 @@ module Admin::FilterHelper
   REPORT_FILTERS       = %i(resolved account_id target_account_id).freeze
   INVITE_FILTER        = %i(available expired).freeze
   CUSTOM_EMOJI_FILTERS = %i(local remote by_domain shortcode).freeze
-  TAGS_FILTERS         = %i(hidden).freeze
+  TAGS_FILTERS         = %i(context review).freeze
   INSTANCES_FILTERS    = %i(limited by_domain).freeze
   FOLLOWERS_FILTERS    = %i(relationship status by_domain activity order).freeze
 
   FILTERS = ACCOUNT_FILTERS + REPORT_FILTERS + INVITE_FILTER + CUSTOM_EMOJI_FILTERS + TAGS_FILTERS + INSTANCES_FILTERS + FOLLOWERS_FILTERS
 
   def filter_link_to(text, link_to_params, link_class_params = link_to_params)
-    new_url = filtered_url_for(link_to_params)
+    new_url   = filtered_url_for(link_to_params)
     new_class = filtered_url_for(link_class_params)
+
     link_to text, new_url, class: filter_link_class(new_class)
   end
 
diff --git a/app/mailers/admin_mailer.rb b/app/mailers/admin_mailer.rb
index 9ab3e2bbd..8abce5f05 100644
--- a/app/mailers/admin_mailer.rb
+++ b/app/mailers/admin_mailer.rb
@@ -24,4 +24,14 @@ class AdminMailer < ApplicationMailer
       mail to: @me.user_email, subject: I18n.t('admin_mailer.new_pending_account.subject', instance: @instance, username: @account.username)
     end
   end
+
+  def new_trending_tag(recipient, tag)
+    @tag      = tag
+    @me       = recipient
+    @instance = Rails.configuration.x.local_domain
+
+    locale_for_account(@me) do
+      mail to: @me.user_email, subject: I18n.t('admin_mailer.new_trending_tag.subject', instance: @instance, name: @tag.name)
+    end
+  end
 end
diff --git a/app/models/application_record.rb b/app/models/application_record.rb
index 83134d41a..c1b873da6 100644
--- a/app/models/application_record.rb
+++ b/app/models/application_record.rb
@@ -2,5 +2,16 @@
 
 class ApplicationRecord < ActiveRecord::Base
   self.abstract_class = true
+
   include Remotable
+
+  def boolean_with_default(key, default_value)
+    value = attributes[key]
+
+    if value.nil?
+      default_value
+    else
+      value
+    end
+  end
 end
diff --git a/app/models/tag.rb b/app/models/tag.rb
index c7f0af86d..6a02581fa 100644
--- a/app/models/tag.rb
+++ b/app/models/tag.rb
@@ -3,11 +3,16 @@
 #
 # Table name: tags
 #
-#  id         :bigint(8)        not null, primary key
-#  name       :string           default(""), not null
-#  created_at :datetime         not null
-#  updated_at :datetime         not null
-#  score      :integer
+#  id                  :bigint(8)        not null, primary key
+#  name                :string           default(""), not null
+#  created_at          :datetime         not null
+#  updated_at          :datetime         not null
+#  score               :integer
+#  usable              :boolean
+#  trendable           :boolean
+#  listable            :boolean
+#  reviewed_at         :datetime
+#  requested_review_at :datetime
 #
 
 class Tag < ApplicationRecord
@@ -22,16 +27,17 @@ class Tag < ApplicationRecord
   HASHTAG_RE = /(?:^|[^\/\)\w])#(#{HASHTAG_NAME_RE})/i
 
   validates :name, presence: true, format: { with: /\A(#{HASHTAG_NAME_RE})\z/i }
+  validate :validate_name_change, if: -> { !new_record? && name_changed? }
 
-  scope :discoverable, -> { joins(:account_tag_stat).where(AccountTagStat.arel_table[:accounts_count].gt(0)).where(account_tag_stats: { hidden: false }).order(Arel.sql('account_tag_stats.accounts_count desc')) }
-  scope :hidden, -> { where(account_tag_stats: { hidden: true }) }
+  scope :reviewed, -> { where.not(reviewed_at: nil) }
+  scope :pending_review, -> { where(reviewed_at: nil).where.not(requested_review_at: nil) }
+  scope :discoverable, -> { where.not(listable: false).joins(:account_tag_stat).where(AccountTagStat.arel_table[:accounts_count].gt(0)).order(Arel.sql('account_tag_stats.accounts_count desc')) }
   scope :most_used, ->(account) { joins(:statuses).where(statuses: { account: account }).group(:id).order(Arel.sql('count(*) desc')) }
 
   delegate :accounts_count,
            :accounts_count=,
            :increment_count!,
            :decrement_count!,
-           :hidden?,
            to: :account_tag_stat
 
   after_save :save_account_tag_stat
@@ -48,6 +54,40 @@ class Tag < ApplicationRecord
     name
   end
 
+  def usable
+    boolean_with_default('usable', true)
+  end
+
+  alias usable? usable
+
+  def listable
+    boolean_with_default('listable', true)
+  end
+
+  alias listable? listable
+
+  def trendable
+    boolean_with_default('trendable', false)
+  end
+
+  alias trendable? trendable
+
+  def requires_review?
+    reviewed_at.nil?
+  end
+
+  def reviewed?
+    reviewed_at.present?
+  end
+
+  def requested_review?
+    requested_review_at.present?
+  end
+
+  def trending?
+    TrendingTags.trending?(self)
+  end
+
   def history
     days = []
 
@@ -117,4 +157,8 @@ class Tag < ApplicationRecord
     return unless account_tag_stat&.changed?
     account_tag_stat.save
   end
+
+  def validate_name_change
+    errors.add(:name, I18n.t('tags.does_not_match_previous_name')) unless name_was.mb_chars.casecmp(name.mb_chars).zero?
+  end
 end
diff --git a/app/models/trending_tags.rb b/app/models/trending_tags.rb
index 211c8f1dc..e9b9b25e3 100644
--- a/app/models/trending_tags.rb
+++ b/app/models/trending_tags.rb
@@ -10,20 +10,28 @@ class TrendingTags
     include Redisable
 
     def record_use!(tag, account, at_time = Time.now.utc)
-      return if disallowed_hashtags.include?(tag.name) || account.silenced? || account.bot?
+      return if account.silenced? || account.bot? || !tag.usable? || !(tag.trendable? || tag.requires_review?)
 
       increment_historical_use!(tag.id, at_time)
       increment_unique_use!(tag.id, account.id, at_time)
-      increment_vote!(tag.id, at_time)
+      increment_vote!(tag, at_time)
     end
 
-    def get(limit)
-      key     = "#{KEY}:#{Time.now.utc.beginning_of_day.to_i}"
-      tag_ids = redis.zrevrange(key, 0, limit - 1).map(&:to_i)
-      tags    = Tag.where(id: tag_ids).to_a.each_with_object({}) { |tag, h| h[tag.id] = tag }
+    def get(limit, filtered: true)
+      tag_ids = redis.zrevrange("#{KEY}:#{Time.now.utc.beginning_of_day.to_i}", 0, limit - 1).map(&:to_i)
+
+      tags = Tag.where(id: tag_ids)
+      tags = tags.where(trendable: true) if filtered
+      tags = tags.each_with_object({}) { |tag, h| h[tag.id] = tag }
+
       tag_ids.map { |tag_id| tags[tag_id] }.compact
     end
 
+    def trending?(tag)
+      rank = redis.zrevrank("#{KEY}:#{Time.now.utc.beginning_of_day.to_i}", tag.id)
+      rank.present? && rank <= 10
+    end
+
     private
 
     def increment_historical_use!(tag_id, at_time)
@@ -38,33 +46,27 @@ class TrendingTags
       redis.expire(key, EXPIRE_HISTORY_AFTER)
     end
 
-    def increment_vote!(tag_id, at_time)
+    def increment_vote!(tag, at_time)
       key      = "#{KEY}:#{at_time.beginning_of_day.to_i}"
-      expected = redis.pfcount("activity:tags:#{tag_id}:#{(at_time - 1.day).beginning_of_day.to_i}:accounts").to_f
+      expected = redis.pfcount("activity:tags:#{tag.id}:#{(at_time - 1.day).beginning_of_day.to_i}:accounts").to_f
       expected = 1.0 if expected.zero?
-      observed = redis.pfcount("activity:tags:#{tag_id}:#{at_time.beginning_of_day.to_i}:accounts").to_f
+      observed = redis.pfcount("activity:tags:#{tag.id}:#{at_time.beginning_of_day.to_i}:accounts").to_f
 
       if expected > observed || observed < THRESHOLD
-        redis.zrem(key, tag_id.to_s)
+        redis.zrem(key, tag.id)
       else
-        score = ((observed - expected)**2) / expected
-        added = redis.zadd(key, score, tag_id.to_s)
-        bump_tag_score!(tag_id) if added
+        score    = ((observed - expected)**2) / expected
+        old_rank = redis.zrevrank(key, tag.id)
+
+        redis.zadd(key, score, tag.id)
+        request_review!(tag) if (old_rank.nil? || old_rank > 10) && redis.zrevrank(key, tag.id) <= 10 && !tag.trendable? && tag.requires_review? && !tag.requested_review?
       end
 
       redis.expire(key, EXPIRE_TRENDS_AFTER)
     end
 
-    def bump_tag_score!(tag_id)
-      Tag.where(id: tag_id).update_all('score = COALESCE(score, 0) + 1')
-    end
-
-    def disallowed_hashtags
-      return @disallowed_hashtags if defined?(@disallowed_hashtags)
-
-      @disallowed_hashtags = Setting.disallowed_hashtags.nil? ? [] : Setting.disallowed_hashtags
-      @disallowed_hashtags = @disallowed_hashtags.split(' ') if @disallowed_hashtags.is_a? String
-      @disallowed_hashtags = @disallowed_hashtags.map(&:downcase)
+    def request_review!(tag)
+      User.staff.includes(:account).find_each { |u| AdminMailer.new_trending_tag(u.account, tag).deliver_later! if u.allows_trending_tag_emails? }
     end
   end
 end
diff --git a/app/models/user.rb b/app/models/user.rb
index 6806c0362..b83e26af3 100644
--- a/app/models/user.rb
+++ b/app/models/user.rb
@@ -207,6 +207,10 @@ class User < ApplicationRecord
     settings.notification_emails['pending_account']
   end
 
+  def allows_trending_tag_emails?
+    settings.notification_emails['trending_tag']
+  end
+
   def hides_network?
     @hides_network ||= settings.hide_network
   end
diff --git a/app/policies/tag_policy.rb b/app/policies/tag_policy.rb
index c63de01db..aaf70fcab 100644
--- a/app/policies/tag_policy.rb
+++ b/app/policies/tag_policy.rb
@@ -5,11 +5,11 @@ class TagPolicy < ApplicationPolicy
     staff?
   end
 
-  def hide?
+  def show?
     staff?
   end
 
-  def unhide?
+  def update?
     staff?
   end
 end
diff --git a/app/validators/disallowed_hashtags_validator.rb b/app/validators/disallowed_hashtags_validator.rb
index ee06b20f6..d745b767f 100644
--- a/app/validators/disallowed_hashtags_validator.rb
+++ b/app/validators/disallowed_hashtags_validator.rb
@@ -4,24 +4,7 @@ class DisallowedHashtagsValidator < ActiveModel::Validator
   def validate(status)
     return unless status.local? && !status.reblog?
 
-    @status = status
-    tags    = select_tags
-
-    status.errors.add(:text, I18n.t('statuses.disallowed_hashtags', tags: tags.join(', '), count: tags.size)) unless tags.empty?
-  end
-
-  private
-
-  def select_tags
-    tags = Extractor.extract_hashtags(@status.text)
-    tags.keep_if { |tag| disallowed_hashtags.include? tag.downcase }
-  end
-
-  def disallowed_hashtags
-    return @disallowed_hashtags if @disallowed_hashtags
-
-    @disallowed_hashtags = Setting.disallowed_hashtags.nil? ? [] : Setting.disallowed_hashtags
-    @disallowed_hashtags = @disallowed_hashtags.split(' ') if @disallowed_hashtags.is_a? String
-    @disallowed_hashtags = @disallowed_hashtags.map(&:downcase)
+    disallowed_hashtags = Tag.matching_name(Extractor.extract_hashtags(status.text)).reject(&:usable?)
+    status.errors.add(:text, I18n.t('statuses.disallowed_hashtags', tags: disallowed_hashtags.map(&:name).join(', '), count: disallowed_hashtags.size)) unless disallowed_hashtags.empty?
   end
 end
diff --git a/app/views/admin/dashboard/index.html.haml b/app/views/admin/dashboard/index.html.haml
index 77cc1a2a0..910896075 100644
--- a/app/views/admin/dashboard/index.html.haml
+++ b/app/views/admin/dashboard/index.html.haml
@@ -107,5 +107,5 @@
       %ul
         - @trending_hashtags.each do |tag|
           %li
-            = link_to "##{tag.name}", web_url("timelines/tag/#{tag.name}")
+            = link_to content_tag(:span, "##{tag.name}", class: !tag.trendable? && !tag.reviewed? ? 'warning-hint' : (!tag.trendable? ? 'negative-hint' : nil)), admin_tag_path(tag.id)
             %span.pull-right= number_with_delimiter(tag.history[0][:accounts].to_i)
diff --git a/app/views/admin/tags/_tag.html.haml b/app/views/admin/tags/_tag.html.haml
index 961b83f93..91af8e492 100644
--- a/app/views/admin/tags/_tag.html.haml
+++ b/app/views/admin/tags/_tag.html.haml
@@ -1,12 +1,16 @@
-%tr
-  %td
-    = link_to explore_hashtag_path(tag) do
+.directory__tag
+  = link_to admin_tag_path(tag.id) do
+    %h4
       = fa_icon 'hashtag'
       = tag.name
-  %td
-    = t('directories.people', count: tag.accounts_count)
-  %td
-    - if tag.hidden?
-      = table_link_to 'eye', t('admin.tags.unhide'), unhide_admin_tag_path(tag.id, **@filter_params), method: :post
-    - else
-      = table_link_to 'eye-slash', t('admin.tags.hide'), hide_admin_tag_path(tag.id, **@filter_params), method: :post
+
+      %small
+        = t('admin.tags.in_directory', count: tag.accounts_count)
+        &bull;
+        = t('admin.tags.unique_uses_today', count: tag.history.first[:accounts])
+
+        - if tag.trending?
+          = fa_icon 'fire fw'
+          = t('admin.tags.trending_right_now')
+
+    .trends__item__current= number_to_human tag.history.first[:uses], strip_insignificant_zeros: true
diff --git a/app/views/admin/tags/index.html.haml b/app/views/admin/tags/index.html.haml
index 4ba395860..5e4ee21f5 100644
--- a/app/views/admin/tags/index.html.haml
+++ b/app/views/admin/tags/index.html.haml
@@ -3,17 +3,19 @@
 
 .filters
   .filter-subset
-    %strong= t('admin.reports.status')
+    %strong= t('admin.tags.context')
     %ul
-      %li= filter_link_to t('admin.tags.visible'), hidden: nil
-      %li= filter_link_to t('admin.tags.hidden'), hidden: '1'
+      %li= filter_link_to t('generic.all'), context: nil
+      %li= filter_link_to t('admin.tags.directory'), context: 'directory'
 
-.table-wrapper
-  %table.table
-    %thead
-      %tr
-        %th= t('admin.tags.name')
-        %th= t('admin.tags.accounts')
-        %th
-    %tbody
-      = render @tags
+  .filter-subset
+    %strong= t('admin.tags.review')
+    %ul
+      %li= filter_link_to t('generic.all'), review: nil
+      %li= filter_link_to t('admin.tags.reviewed'), review: 'reviewed'
+      %li= filter_link_to safe_join([t('admin.accounts.moderation.pending'), "(#{Tag.pending_review.count})"], ' '), review: 'pending_review'
+
+%hr.spacer/
+
+= render @tags
+= paginate @tags
diff --git a/app/views/admin/tags/show.html.haml b/app/views/admin/tags/show.html.haml
new file mode 100644
index 000000000..27c8dc92b
--- /dev/null
+++ b/app/views/admin/tags/show.html.haml
@@ -0,0 +1,16 @@
+- content_for :page_title do
+  = "##{@tag.name}"
+
+= simple_form_for @tag, url: admin_tag_path(@tag.id) do |f|
+  = render 'shared/error_messages', object: @tag
+
+  .fields-group
+    = f.input :name, wrapper: :with_block_label
+
+  .fields-group
+    = f.input :usable, as: :boolean, wrapper: :with_label
+    = f.input :trendable, as: :boolean, wrapper: :with_label
+    = f.input :listable, as: :boolean, wrapper: :with_label
+
+  .actions
+    = f.button :button, t('generic.save_changes'), type: :submit
diff --git a/app/views/admin_mailer/new_trending_tag.text.erb b/app/views/admin_mailer/new_trending_tag.text.erb
new file mode 100644
index 000000000..f3087df37
--- /dev/null
+++ b/app/views/admin_mailer/new_trending_tag.text.erb
@@ -0,0 +1,5 @@
+<%= raw t('application_mailer.salutation', name: display_name(@me)) %>
+
+<%= raw t('admin_mailer.new_trending_tag.body', name: @tag.name) %>
+
+<%= raw t('application_mailer.view')%> <%= admin_tags_url(review: 'pending_review') %>
diff --git a/app/views/settings/preferences/notifications/show.html.haml b/app/views/settings/preferences/notifications/show.html.haml
index acc646fc3..f666ae4ff 100644
--- a/app/views/settings/preferences/notifications/show.html.haml
+++ b/app/views/settings/preferences/notifications/show.html.haml
@@ -15,6 +15,7 @@
       - if current_user.staff?
         = ff.input :report, as: :boolean, wrapper: :with_label
         = ff.input :pending_account, as: :boolean, wrapper: :with_label
+        = ff.input :trending_tag, as: :boolean, wrapper: :with_label
 
   .fields-group
     = f.simple_fields_for :notification_emails, hash_to_object(current_user.settings.notification_emails) do |ff|
diff --git a/config/locales/en.yml b/config/locales/en.yml
index 6c1a34300..9b62aac3a 100644
--- a/config/locales/en.yml
+++ b/config/locales/en.yml
@@ -483,13 +483,14 @@ en:
       title: Account statuses
       with_media: With media
     tags:
-      accounts: Accounts
-      hidden: Hidden
-      hide: Hide from directory
-      name: Hashtag
+      context: Context
+      directory: In directory
+      in_directory: "%{count} in directory"
+      review: Review status
+      reviewed: Reviewed
       title: Hashtags
-      unhide: Show in directory
-      visible: Visible
+      trending_right_now: Trending right now
+      unique_uses_today: "%{count} posting today"
     title: Administration
     warning_presets:
       add_new: Add new
@@ -505,6 +506,9 @@ en:
       body: "%{reporter} has reported %{target}"
       body_remote: Someone from %{domain} has reported %{target}
       subject: New report for %{instance} (#%{id})
+    new_trending_tag:
+      body: 'The hashtag #%{name} is trending today, but has not been previously reviewed. It will not be displayed publicly unless you allow it to, or just save the form as it is to never hear about it again.'
+      subject: New hashtag up for review on %{instance} (#%{name})
   appearance:
     advanced_web_interface: Advanced web interface
     advanced_web_interface_hint: 'If you want to make use of your entire screen width, the advanced web interface allows you to configure many different columns to see as much information at the same time as you want: Home, notifications, federated timeline, any number of lists and hashtags.'
@@ -939,6 +943,8 @@ en:
     pinned: Pinned toot
     reblogged: boosted
     sensitive_content: Sensitive content
+  tags:
+    does_not_match_previous_name: does not match the previous name
   terms:
     body_html: |
       <h2>Privacy Policy</h2>
diff --git a/config/locales/simple_form.en.yml b/config/locales/simple_form.en.yml
index 10b30e627..6fdfc9d7b 100644
--- a/config/locales/simple_form.en.yml
+++ b/config/locales/simple_form.en.yml
@@ -48,6 +48,8 @@ en:
         text: This will help us review your application
       sessions:
         otp: 'Enter the two-factor code generated by your phone app or use one of your recovery codes:'
+      tag:
+        name: You can only change the casing of the letters, for example, to make it more readable
       user:
         chosen_languages: When checked, only toots in selected languages will be displayed in public timelines
     labels:
@@ -137,6 +139,11 @@ en:
         pending_account: Send e-mail when a new account needs review
         reblog: Send e-mail when someone boosts your status
         report: Send e-mail when a new report is submitted
+        trending_tag: Send e-mail when an unreviewed hashtag is trending
+      tag:
+        listable: Allow this hashtag to appear on the profile directory
+        trendable: Allow this hashtag to appear under trends
+        usable: Allow toots to use this hashtag
     'no': 'No'
     recommended: Recommended
     required:
diff --git a/config/navigation.rb b/config/navigation.rb
index 9b46da603..38668bbf7 100644
--- a/config/navigation.rb
+++ b/config/navigation.rb
@@ -38,7 +38,7 @@ SimpleNavigation::Configuration.run do |navigation|
       s.item :reports, safe_join([fa_icon('flag fw'), t('admin.reports.title')]), admin_reports_url, highlights_on: %r{/admin/reports}
       s.item :accounts, safe_join([fa_icon('users fw'), t('admin.accounts.title')]), admin_accounts_url, highlights_on: %r{/admin/accounts|/admin/pending_accounts}
       s.item :invites, safe_join([fa_icon('user-plus fw'), t('admin.invites.title')]), admin_invites_path
-      s.item :tags, safe_join([fa_icon('tag fw'), t('admin.tags.title')]), admin_tags_path
+      s.item :tags, safe_join([fa_icon('hashtag fw'), t('admin.tags.title')]), admin_tags_path, highlights_on: %r{/admin/tags}
       s.item :instances, safe_join([fa_icon('cloud fw'), t('admin.instances.title')]), admin_instances_url(limited: whitelist_mode? ? nil : '1'), highlights_on: %r{/admin/instances|/admin/domain_blocks|/admin/domain_allows}, if: -> { current_user.admin? }
       s.item :email_domain_blocks, safe_join([fa_icon('envelope fw'), t('admin.email_domain_blocks.title')]), admin_email_domain_blocks_url, highlights_on: %r{/admin/email_domain_blocks}, if: -> { current_user.admin? }
     end
diff --git a/config/routes.rb b/config/routes.rb
index 04424bbbd..60f7d2e05 100644
--- a/config/routes.rb
+++ b/config/routes.rb
@@ -243,13 +243,7 @@ Rails.application.routes.draw do
     end
 
     resources :account_moderation_notes, only: [:create, :destroy]
-
-    resources :tags, only: [:index] do
-      member do
-        post :hide
-        post :unhide
-      end
-    end
+    resources :tags, only: [:index, :show, :update]
   end
 
   get '/admin', to: redirect('/admin/dashboard', status: 302)
@@ -311,6 +305,7 @@ Rails.application.routes.draw do
       resources :mutes,        only: [:index]
       resources :favourites,   only: [:index]
       resources :reports,      only: [:create]
+      resources :trends,       only: [:index]
       resources :filters,      only: [:index, :create, :show, :update, :destroy]
       resources :endorsements, only: [:index]
 
diff --git a/config/settings.yml b/config/settings.yml
index ad2970bb7..10180201f 100644
--- a/config/settings.yml
+++ b/config/settings.yml
@@ -43,6 +43,7 @@ defaults: &defaults
     digest: true
     report: true
     pending_account: true
+    trending_tag: true
   interactions:
     must_be_follower: false
     must_be_following: false
diff --git a/db/migrate/20190805123746_add_capabilities_to_tags.rb b/db/migrate/20190805123746_add_capabilities_to_tags.rb
new file mode 100644
index 000000000..43c7763b1
--- /dev/null
+++ b/db/migrate/20190805123746_add_capabilities_to_tags.rb
@@ -0,0 +1,9 @@
+class AddCapabilitiesToTags < ActiveRecord::Migration[5.2]
+  def change
+    add_column :tags, :usable, :boolean
+    add_column :tags, :trendable, :boolean
+    add_column :tags, :listable, :boolean
+    add_column :tags, :reviewed_at, :datetime
+    add_column :tags, :requested_review_at, :datetime
+  end
+end
diff --git a/db/schema.rb b/db/schema.rb
index e3af9c31a..d1b6825b4 100644
--- a/db/schema.rb
+++ b/db/schema.rb
@@ -10,7 +10,7 @@
 #
 # It's strongly recommended that you check this file into your version control system.
 
-ActiveRecord::Schema.define(version: 2019_07_29_185330) do
+ActiveRecord::Schema.define(version: 2019_08_05_123746) do
 
   # These are extensions that must be enabled in order to support this database
   enable_extension "plpgsql"
@@ -660,6 +660,11 @@ ActiveRecord::Schema.define(version: 2019_07_29_185330) do
     t.datetime "created_at", null: false
     t.datetime "updated_at", null: false
     t.integer "score"
+    t.boolean "usable"
+    t.boolean "trendable"
+    t.boolean "listable"
+    t.datetime "reviewed_at"
+    t.datetime "requested_review_at"
     t.index "lower((name)::text)", name: "index_tags_on_name_lower", unique: true
   end
 
diff --git a/spec/controllers/admin/tags_controller_spec.rb b/spec/controllers/admin/tags_controller_spec.rb
index 3af994071..5c1944fc7 100644
--- a/spec/controllers/admin/tags_controller_spec.rb
+++ b/spec/controllers/admin/tags_controller_spec.rb
@@ -10,62 +10,14 @@ RSpec.describe Admin::TagsController, type: :controller do
   end
 
   describe 'GET #index' do
-    before do
-      account_tag_stat = Fabricate(:tag).account_tag_stat
-      account_tag_stat.update(hidden: hidden, accounts_count: 1)
-      get :index, params: { hidden: hidden }
-    end
-
-    context 'with hidden tags' do
-      let(:hidden) { true }
-
-      it 'returns status 200' do
-        expect(response).to have_http_status(200)
-      end
-    end
-
-    context 'without hidden tags' do
-      let(:hidden) { false }
-
-      it 'returns status 200' do
-        expect(response).to have_http_status(200)
-      end
-    end
-  end
-
-  describe 'POST #hide' do
-    let(:tag) { Fabricate(:tag) }
+    let!(:tag) { Fabricate(:tag) }
 
     before do
-      tag.account_tag_stat.update(hidden: false)
-      post :hide, params: { id: tag.id }
-    end
-
-    it 'hides tag' do
-      tag.reload
-      expect(tag).to be_hidden
-    end
-
-    it 'redirects to admin_tags_path' do
-      expect(response).to redirect_to(admin_tags_path(controller.instance_variable_get(:@filter_params)))
-    end
-  end
-
-  describe 'POST #unhide' do
-    let(:tag) { Fabricate(:tag) }
-
-    before do
-      tag.account_tag_stat.update(hidden: true)
-      post :unhide, params: { id: tag.id }
-    end
-
-    it 'unhides tag' do
-      tag.reload
-      expect(tag).not_to be_hidden
+      get :index
     end
 
-    it 'redirects to admin_tags_path' do
-      expect(response).to redirect_to(admin_tags_path(controller.instance_variable_get(:@filter_params)))
+    it 'returns status 200' do
+      expect(response).to have_http_status(200)
     end
   end
 end
diff --git a/spec/policies/tag_policy_spec.rb b/spec/policies/tag_policy_spec.rb
index c7afaa7c9..c63875dc0 100644
--- a/spec/policies/tag_policy_spec.rb
+++ b/spec/policies/tag_policy_spec.rb
@@ -8,7 +8,7 @@ RSpec.describe TagPolicy do
   let(:admin)   { Fabricate(:user, admin: true).account }
   let(:john)    { Fabricate(:user).account }
 
-  permissions :index?, :hide?, :unhide? do
+  permissions :index?, :show?, :update? do
     context 'staff?' do
       it 'permits' do
         expect(subject).to permit(admin, Tag)
diff --git a/spec/validators/disallowed_hashtags_validator_spec.rb b/spec/validators/disallowed_hashtags_validator_spec.rb
index 8ec1302ab..9deec0bb9 100644
--- a/spec/validators/disallowed_hashtags_validator_spec.rb
+++ b/spec/validators/disallowed_hashtags_validator_spec.rb
@@ -3,42 +3,44 @@
 require 'rails_helper'
 
 RSpec.describe DisallowedHashtagsValidator, type: :validator do
+  let(:disallowed_tags) { [] }
+
   describe '#validate' do
     before do
-      allow_any_instance_of(described_class).to receive(:select_tags) { tags }
+      disallowed_tags.each { |name| Fabricate(:tag, name: name, usable: false) }
       described_class.new.validate(status)
     end
 
-    let(:status) { double(errors: errors, local?: local, reblog?: reblog, text: '') }
+    let(:status) { double(errors: errors, local?: local, reblog?: reblog, text: disallowed_tags.map { |x| '#' + x }.join(' ')) }
     let(:errors) { double(add: nil) }
 
-    context 'unless status.local? && !status.reblog?' do
+    context 'for a remote reblog' do
       let(:local)  { false }
       let(:reblog) { true }
 
-      it 'not calls errors.add' do
+      it 'does not add errors' do
         expect(errors).not_to have_received(:add).with(:text, any_args)
       end
     end
 
-    context 'status.local? && !status.reblog?' do
+    context 'for a local original status' do
       let(:local)  { true }
       let(:reblog) { false }
 
-      context 'tags.empty?' do
-        let(:tags) { [] }
+      context 'when does not contain any disallowed hashtags' do
+        let(:disallowed_tags) { [] }
 
-        it 'not calls errors.add' do
+        it 'does not add errors' do
           expect(errors).not_to have_received(:add).with(:text, any_args)
         end
       end
 
-      context '!tags.empty?' do
-        let(:tags) { %w(a b c) }
+      context 'when contains disallowed hashtags' do
+        let(:disallowed_tags) { %w(a b c) }
 
-        it 'calls errors.add' do
+        it 'adds an error' do
           expect(errors).to have_received(:add)
-            .with(:text, I18n.t('statuses.disallowed_hashtags', tags: tags.join(', '), count: tags.size))
+            .with(:text, I18n.t('statuses.disallowed_hashtags', tags: disallowed_tags.join(', '), count: disallowed_tags.size))
         end
       end
     end
-- 
cgit 


From 9072fe5ab6464cc9c7a871d388464c7afcf41cd0 Mon Sep 17 00:00:00 2001
From: Eugen Rochko <eugen@zeonfederated.com>
Date: Tue, 6 Aug 2019 17:57:52 +0200
Subject: Add trends UI with admin and user settings (#11502)

---
 app/controllers/settings/preferences_controller.rb |  1 +
 app/javascript/mastodon/actions/trends.js          | 32 ++++++++++++++++
 .../features/getting_started/components/trends.js  | 43 ++++++++++++++++++++++
 .../getting_started/containers/trends_container.js | 13 +++++++
 .../mastodon/features/getting_started/index.js     |  5 ++-
 .../features/ui/components/navigation_panel.js     |  6 ++-
 app/javascript/mastodon/initial_state.js           |  1 +
 app/javascript/mastodon/reducers/index.js          |  2 +
 app/javascript/mastodon/reducers/settings.js       |  4 ++
 app/javascript/mastodon/reducers/trends.js         | 23 ++++++++++++
 app/javascript/styles/mastodon/components.scss     | 38 +++++++++++++++----
 app/lib/user_settings_decorator.rb                 |  5 +++
 app/models/form/admin_settings.rb                  |  2 +
 app/models/trending_tags.rb                        |  4 ++
 app/models/user.rb                                 |  3 +-
 app/serializers/initial_state_serializer.rb        |  2 +
 app/views/admin/settings/edit.html.haml            |  3 ++
 .../settings/preferences/appearance/show.html.haml |  5 +++
 config/locales/en.yml                              |  8 +++-
 config/locales/simple_form.en.yml                  |  1 +
 config/settings.yml                                |  1 +
 21 files changed, 189 insertions(+), 13 deletions(-)
 create mode 100644 app/javascript/mastodon/actions/trends.js
 create mode 100644 app/javascript/mastodon/features/getting_started/components/trends.js
 create mode 100644 app/javascript/mastodon/features/getting_started/containers/trends_container.js
 create mode 100644 app/javascript/mastodon/reducers/trends.js

(limited to 'app/models')

diff --git a/app/controllers/settings/preferences_controller.rb b/app/controllers/settings/preferences_controller.rb
index d548072a8..edf29947b 100644
--- a/app/controllers/settings/preferences_controller.rb
+++ b/app/controllers/settings/preferences_controller.rb
@@ -56,6 +56,7 @@ class Settings::PreferencesController < Settings::BaseController
       :setting_advanced_layout,
       :setting_use_blurhash,
       :setting_use_pending_items,
+      :setting_trends,
       notification_emails: %i(follow follow_request reblog favourite mention digest report pending_account trending_tag),
       interactions: %i(must_be_follower must_be_following must_be_following_dm)
     )
diff --git a/app/javascript/mastodon/actions/trends.js b/app/javascript/mastodon/actions/trends.js
new file mode 100644
index 000000000..853e4f60a
--- /dev/null
+++ b/app/javascript/mastodon/actions/trends.js
@@ -0,0 +1,32 @@
+import api from '../api';
+
+export const TRENDS_FETCH_REQUEST = 'TRENDS_FETCH_REQUEST';
+export const TRENDS_FETCH_SUCCESS = 'TRENDS_FETCH_SUCCESS';
+export const TRENDS_FETCH_FAIL    = 'TRENDS_FETCH_FAIL';
+
+export const fetchTrends = () => (dispatch, getState) => {
+  dispatch(fetchTrendsRequest());
+
+  api(getState)
+    .get('/api/v1/trends')
+    .then(({ data }) => dispatch(fetchTrendsSuccess(data)))
+    .catch(err => dispatch(fetchTrendsFail(err)));
+};
+
+export const fetchTrendsRequest = () => ({
+  type: TRENDS_FETCH_REQUEST,
+  skipLoading: true,
+});
+
+export const fetchTrendsSuccess = trends => ({
+  type: TRENDS_FETCH_SUCCESS,
+  trends,
+  skipLoading: true,
+});
+
+export const fetchTrendsFail = error => ({
+  type: TRENDS_FETCH_FAIL,
+  error,
+  skipLoading: true,
+  skipAlert: true,
+});
diff --git a/app/javascript/mastodon/features/getting_started/components/trends.js b/app/javascript/mastodon/features/getting_started/components/trends.js
new file mode 100644
index 000000000..1dcacc8b3
--- /dev/null
+++ b/app/javascript/mastodon/features/getting_started/components/trends.js
@@ -0,0 +1,43 @@
+import React from 'react';
+import ImmutablePureComponent from 'react-immutable-pure-component';
+import PropTypes from 'prop-types';
+import ImmutablePropTypes from 'react-immutable-proptypes';
+import Hashtag from 'mastodon/components/hashtag';
+
+export default class Trends extends ImmutablePureComponent {
+
+  static defaultProps = {
+    loading: false,
+  };
+
+  static propTypes = {
+    trends: ImmutablePropTypes.list,
+    fetchTrends: PropTypes.func.isRequired,
+  };
+
+  componentDidMount () {
+    this.props.fetchTrends();
+    this.refreshInterval = setInterval(() => this.props.fetchTrends(), 36000);
+  }
+
+  componentWillUnmount () {
+    if (this.refreshInterval) {
+      clearInterval(this.refreshInterval);
+    }
+  }
+
+  render () {
+    const { trends } = this.props;
+
+    if (!trends || trends.isEmpty()) {
+      return null;
+    }
+
+    return (
+      <div className='getting-started__trends'>
+        {trends.take(3).map(hashtag => <Hashtag key={hashtag.get('name')} hashtag={hashtag} />)}
+      </div>
+    );
+  }
+
+}
diff --git a/app/javascript/mastodon/features/getting_started/containers/trends_container.js b/app/javascript/mastodon/features/getting_started/containers/trends_container.js
new file mode 100644
index 000000000..1df3fb4fe
--- /dev/null
+++ b/app/javascript/mastodon/features/getting_started/containers/trends_container.js
@@ -0,0 +1,13 @@
+import { connect } from 'react-redux';
+import { fetchTrends } from '../../../actions/trends';
+import Trends from '../components/trends';
+
+const mapStateToProps = state => ({
+  trends: state.getIn(['trends', 'items']),
+});
+
+const mapDispatchToProps = dispatch => ({
+  fetchTrends: () => dispatch(fetchTrends()),
+});
+
+export default connect(mapStateToProps, mapDispatchToProps)(Trends);
diff --git a/app/javascript/mastodon/features/getting_started/index.js b/app/javascript/mastodon/features/getting_started/index.js
index 791f22d47..6a122a750 100644
--- a/app/javascript/mastodon/features/getting_started/index.js
+++ b/app/javascript/mastodon/features/getting_started/index.js
@@ -7,12 +7,13 @@ import { connect } from 'react-redux';
 import PropTypes from 'prop-types';
 import ImmutablePropTypes from 'react-immutable-proptypes';
 import ImmutablePureComponent from 'react-immutable-pure-component';
-import { me, profile_directory } from '../../initial_state';
+import { me, profile_directory, showTrends } from '../../initial_state';
 import { fetchFollowRequests } from 'mastodon/actions/accounts';
 import { List as ImmutableList } from 'immutable';
 import NavigationBar from '../compose/components/navigation_bar';
 import Icon from 'mastodon/components/icon';
 import LinkFooter from 'mastodon/features/ui/components/link_footer';
+import TrendsContainer from './containers/trends_container';
 
 const messages = defineMessages({
   home_timeline: { id: 'tabs_bar.home', defaultMessage: 'Home' },
@@ -168,6 +169,8 @@ class GettingStarted extends ImmutablePureComponent {
 
           <LinkFooter withHotkeys={multiColumn} />
         </div>
+
+        {multiColumn && showTrends && <TrendsContainer />}
       </Column>
     );
   }
diff --git a/app/javascript/mastodon/features/ui/components/navigation_panel.js b/app/javascript/mastodon/features/ui/components/navigation_panel.js
index ef3ad2e09..64a40a9da 100644
--- a/app/javascript/mastodon/features/ui/components/navigation_panel.js
+++ b/app/javascript/mastodon/features/ui/components/navigation_panel.js
@@ -2,10 +2,11 @@ import React from 'react';
 import { NavLink, withRouter } from 'react-router-dom';
 import { FormattedMessage } from 'react-intl';
 import Icon from 'mastodon/components/icon';
-import { profile_directory } from 'mastodon/initial_state';
+import { profile_directory, showTrends } from 'mastodon/initial_state';
 import NotificationsCounterIcon from './notifications_counter_icon';
 import FollowRequestsNavLink from './follow_requests_nav_link';
 import ListPanel from './list_panel';
+import TrendsContainer from 'mastodon/features/getting_started/containers/trends_container';
 
 const NavigationPanel = () => (
   <div className='navigation-panel'>
@@ -25,6 +26,9 @@ const NavigationPanel = () => (
     <a className='column-link column-link--transparent' href='/settings/preferences'><Icon className='column-link__icon' id='cog' fixedWidth /><FormattedMessage id='navigation_bar.preferences' defaultMessage='Preferences' /></a>
     <a className='column-link column-link--transparent' href='/relationships'><Icon className='column-link__icon' id='users' fixedWidth /><FormattedMessage id='navigation_bar.follows_and_followers' defaultMessage='Follows and followers' /></a>
     {!!profile_directory && <a className='column-link column-link--transparent' href='/explore'><Icon className='column-link__icon' id='address-book-o' fixedWidth /><FormattedMessage id='navigation_bar.profile_directory' defaultMessage='Profile directory' /></a>}
+
+    {showTrends && <div className='flex-spacer' />}
+    {showTrends && <TrendsContainer />}
   </div>
 );
 
diff --git a/app/javascript/mastodon/initial_state.js b/app/javascript/mastodon/initial_state.js
index cb2ccc7c4..38e7b0595 100644
--- a/app/javascript/mastodon/initial_state.js
+++ b/app/javascript/mastodon/initial_state.js
@@ -22,5 +22,6 @@ export const isStaff = getMeta('is_staff');
 export const forceSingleColumn = !getMeta('advanced_layout');
 export const useBlurhash = getMeta('use_blurhash');
 export const usePendingItems = getMeta('use_pending_items');
+export const showTrends = getMeta('trends');
 
 export default initialState;
diff --git a/app/javascript/mastodon/reducers/index.js b/app/javascript/mastodon/reducers/index.js
index 981ad8e64..3b60878eb 100644
--- a/app/javascript/mastodon/reducers/index.js
+++ b/app/javascript/mastodon/reducers/index.js
@@ -31,6 +31,7 @@ import conversations from './conversations';
 import suggestions from './suggestions';
 import polls from './polls';
 import identity_proofs from './identity_proofs';
+import trends from './trends';
 
 const reducers = {
   dropdown_menu,
@@ -65,6 +66,7 @@ const reducers = {
   conversations,
   suggestions,
   polls,
+  trends,
 };
 
 export default combineReducers(reducers);
diff --git a/app/javascript/mastodon/reducers/settings.js b/app/javascript/mastodon/reducers/settings.js
index 033bfc999..793a99f8f 100644
--- a/app/javascript/mastodon/reducers/settings.js
+++ b/app/javascript/mastodon/reducers/settings.js
@@ -12,6 +12,10 @@ const initialState = ImmutableMap({
 
   skinTone: 1,
 
+  trends: ImmutableMap({
+    show: true,
+  }),
+
   home: ImmutableMap({
     shows: ImmutableMap({
       reblog: true,
diff --git a/app/javascript/mastodon/reducers/trends.js b/app/javascript/mastodon/reducers/trends.js
new file mode 100644
index 000000000..5cecc8fca
--- /dev/null
+++ b/app/javascript/mastodon/reducers/trends.js
@@ -0,0 +1,23 @@
+import { TRENDS_FETCH_REQUEST, TRENDS_FETCH_SUCCESS, TRENDS_FETCH_FAIL } from '../actions/trends';
+import { Map as ImmutableMap, List as ImmutableList, fromJS } from 'immutable';
+
+const initialState = ImmutableMap({
+  items: ImmutableList(),
+  isLoading: false,
+});
+
+export default function trendsReducer(state = initialState, action) {
+  switch(action.type) {
+  case TRENDS_FETCH_REQUEST:
+    return state.set('isLoading', true);
+  case TRENDS_FETCH_SUCCESS:
+    return state.withMutations(map => {
+      map.set('items', fromJS(action.trends));
+      map.set('isLoading', false);
+    });
+  case TRENDS_FETCH_FAIL:
+    return state.set('isLoading', false);
+  default:
+    return state;
+  }
+};
diff --git a/app/javascript/styles/mastodon/components.scss b/app/javascript/styles/mastodon/components.scss
index f02458ded..8de72d72e 100644
--- a/app/javascript/styles/mastodon/components.scss
+++ b/app/javascript/styles/mastodon/components.scss
@@ -2212,7 +2212,6 @@ a.account__display-name {
   }
 
   .getting-started__wrapper,
-  .getting-started__trends,
   .search {
     margin-bottom: 10px;
   }
@@ -2319,13 +2318,24 @@ a.account__display-name {
   margin-bottom: 10px;
   height: calc(100% - 20px);
   overflow-y: auto;
+  display: flex;
+  flex-direction: column;
+
+  & > a {
+    flex: 0 0 auto;
+  }
 
   hr {
+    flex: 0 0 auto;
     border: 0;
     background: transparent;
     border-top: 1px solid lighten($ui-base-color, 4%);
     margin: 10px 0;
   }
+
+  .flex-spacer {
+    background: transparent;
+  }
 }
 
 .drawer__pager {
@@ -2717,8 +2727,10 @@ a.account__display-name {
   }
 
   &__trends {
-    background: $ui-base-color;
     flex: 0 1 auto;
+    opacity: 1;
+    animation: fade 150ms linear;
+    margin-top: 10px;
 
     @media screen and (max-height: 810px) {
       .trends__item:nth-child(3) {
@@ -2735,11 +2747,15 @@ a.account__display-name {
     @media screen and (max-height: 670px) {
       display: none;
     }
-  }
 
-  &__scrollable {
-    max-height: 100%;
-    overflow-y: auto;
+    .trends__item {
+      border-bottom: 0;
+      padding: 10px;
+
+      &__current {
+        color: $darker-text-color;
+      }
+    }
   }
 }
 
@@ -5968,7 +5984,8 @@ noscript {
       font-size: 24px;
       line-height: 36px;
       font-weight: 500;
-      text-align: center;
+      text-align: right;
+      padding-right: 15px;
       color: $secondary-text-color;
     }
 
@@ -5976,7 +5993,12 @@ noscript {
       flex: 0 0 auto;
       width: 50px;
 
-      path {
+      path:first-child {
+        fill: rgba($highlight-text-color, 0.25) !important;
+        fill-opacity: 1 !important;
+      }
+
+      path:last-child {
         stroke: lighten($highlight-text-color, 6%) !important;
       }
     }
diff --git a/app/lib/user_settings_decorator.rb b/app/lib/user_settings_decorator.rb
index 9ae9986c2..3568a3e11 100644
--- a/app/lib/user_settings_decorator.rb
+++ b/app/lib/user_settings_decorator.rb
@@ -36,6 +36,7 @@ class UserSettingsDecorator
     user.settings['advanced_layout']     = advanced_layout_preference if change?('setting_advanced_layout')
     user.settings['use_blurhash']        = use_blurhash_preference if change?('setting_use_blurhash')
     user.settings['use_pending_items']   = use_pending_items_preference if change?('setting_use_pending_items')
+    user.settings['trends']              = trends_preference if change?('setting_trends')
   end
 
   def merged_notification_emails
@@ -122,6 +123,10 @@ class UserSettingsDecorator
     boolean_cast_setting 'setting_use_pending_items'
   end
 
+  def trends_preference
+    boolean_cast_setting 'setting_trends'
+  end
+
   def boolean_cast_setting(key)
     ActiveModel::Type::Boolean.new.cast(settings[key])
   end
diff --git a/app/models/form/admin_settings.rb b/app/models/form/admin_settings.rb
index 2c03c88a8..051268375 100644
--- a/app/models/form/admin_settings.rb
+++ b/app/models/form/admin_settings.rb
@@ -29,6 +29,7 @@ class Form::AdminSettings
     hero
     mascot
     spam_check_enabled
+    trends
   ).freeze
 
   BOOLEAN_KEYS = %i(
@@ -41,6 +42,7 @@ class Form::AdminSettings
     preview_sensitive_media
     profile_directory
     spam_check_enabled
+    trends
   ).freeze
 
   UPLOAD_KEYS = %i(
diff --git a/app/models/trending_tags.rb b/app/models/trending_tags.rb
index e9b9b25e3..0a7e2feac 100644
--- a/app/models/trending_tags.rb
+++ b/app/models/trending_tags.rb
@@ -66,6 +66,10 @@ class TrendingTags
     end
 
     def request_review!(tag)
+      return unless Setting.trends
+
+      tag.touch(:requested_review_at)
+
       User.staff.includes(:account).find_each { |u| AdminMailer.new_trending_tag(u.account, tag).deliver_later! if u.allows_trending_tag_emails? }
     end
   end
diff --git a/app/models/user.rb b/app/models/user.rb
index b83e26af3..a4a20d975 100644
--- a/app/models/user.rb
+++ b/app/models/user.rb
@@ -107,7 +107,8 @@ class User < ApplicationRecord
   delegate :auto_play_gif, :default_sensitive, :unfollow_modal, :boost_modal, :delete_modal,
            :reduce_motion, :system_font_ui, :noindex, :theme, :display_media, :hide_network,
            :expand_spoilers, :default_language, :aggregate_reblogs, :show_application,
-           :advanced_layout, :use_blurhash, :use_pending_items, to: :settings, prefix: :setting, allow_nil: false
+           :advanced_layout, :use_blurhash, :use_pending_items, :trends,
+           to: :settings, prefix: :setting, allow_nil: false
 
   attr_reader :invite_code
   attr_writer :external
diff --git a/app/serializers/initial_state_serializer.rb b/app/serializers/initial_state_serializer.rb
index 7e5d3eda9..c92c5e606 100644
--- a/app/serializers/initial_state_serializer.rb
+++ b/app/serializers/initial_state_serializer.rb
@@ -20,6 +20,7 @@ class InitialStateSerializer < ActiveModel::Serializer
       invites_enabled: Setting.min_invite_role == 'user',
       mascot: instance_presenter.mascot&.file&.url,
       profile_directory: Setting.profile_directory,
+      trends: Setting.trends,
     }
 
     if object.current_account
@@ -35,6 +36,7 @@ class InitialStateSerializer < ActiveModel::Serializer
       store[:use_blurhash]      = object.current_account.user.setting_use_blurhash
       store[:use_pending_items] = object.current_account.user.setting_use_pending_items
       store[:is_staff]          = object.current_account.user.staff?
+      store[:trends]            = Setting.trends && object.current_account.user.setting_trends
     end
 
     store
diff --git a/app/views/admin/settings/edit.html.haml b/app/views/admin/settings/edit.html.haml
index 1e2ed3f77..28c0ece15 100644
--- a/app/views/admin/settings/edit.html.haml
+++ b/app/views/admin/settings/edit.html.haml
@@ -68,6 +68,9 @@
     .fields-group
       = f.input :profile_directory, as: :boolean, wrapper: :with_label, label: t('admin.settings.profile_directory.title'), hint: t('admin.settings.profile_directory.desc_html')
 
+    .fields-group
+      = f.input :trends, as: :boolean, wrapper: :with_label, label: t('admin.settings.trends.title'), hint: t('admin.settings.trends.desc_html')
+
   .fields-group
     = f.input :spam_check_enabled, as: :boolean, wrapper: :with_label, label: t('admin.settings.spam_check_enabled.title'), hint: t('admin.settings.spam_check_enabled.desc_html')
 
diff --git a/app/views/settings/preferences/appearance/show.html.haml b/app/views/settings/preferences/appearance/show.html.haml
index e279a61c4..d6ee1933f 100644
--- a/app/views/settings/preferences/appearance/show.html.haml
+++ b/app/views/settings/preferences/appearance/show.html.haml
@@ -25,6 +25,11 @@
     = f.input :setting_reduce_motion, as: :boolean, wrapper: :with_label
     = f.input :setting_system_font_ui, as: :boolean, wrapper: :with_label
 
+  %h4= t 'appearance.discovery'
+
+  .fields-group
+    = f.input :setting_trends, as: :boolean, wrapper: :with_label
+
   %h4= t 'appearance.confirmation_dialogs'
 
   .fields-group
diff --git a/config/locales/en.yml b/config/locales/en.yml
index 9b62aac3a..67c392662 100644
--- a/config/locales/en.yml
+++ b/config/locales/en.yml
@@ -460,8 +460,8 @@ en:
         title: Custom terms of service
       site_title: Server name
       spam_check_enabled:
-        desc_html: Mastodon can auto-silence and auto-report accounts based on measures such as detecting accounts who send repeated unsolicited messages. There may be false positives.
-        title: Anti-spam
+        desc_html: Mastodon can auto-silence and auto-report accounts that send repeated unsolicited messages. There may be false positives.
+        title: Anti-spam automation
       thumbnail:
         desc_html: Used for previews via OpenGraph and API. 1200x630px recommended
         title: Server thumbnail
@@ -469,6 +469,9 @@ en:
         desc_html: Display public timeline on landing page
         title: Timeline preview
       title: Site settings
+      trends:
+        desc_html: Publicly display previously reviewed hashtags that are currently trending
+        title: Trending hashtags
     statuses:
       back_to_account: Back to account page
       batch:
@@ -514,6 +517,7 @@ en:
     advanced_web_interface_hint: 'If you want to make use of your entire screen width, the advanced web interface allows you to configure many different columns to see as much information at the same time as you want: Home, notifications, federated timeline, any number of lists and hashtags.'
     animations_and_accessibility: Animations and accessibility
     confirmation_dialogs: Confirmation dialogs
+    discovery: Discovery
     sensitive_content: Sensitive content
   application_mailer:
     notification_preferences: Change e-mail preferences
diff --git a/config/locales/simple_form.en.yml b/config/locales/simple_form.en.yml
index 6fdfc9d7b..e15d5904f 100644
--- a/config/locales/simple_form.en.yml
+++ b/config/locales/simple_form.en.yml
@@ -114,6 +114,7 @@ en:
         setting_show_application: Disclose application used to send toots
         setting_system_font_ui: Use system's default font
         setting_theme: Site theme
+        setting_trends: Show today's trends
         setting_unfollow_modal: Show confirmation dialog before unfollowing someone
         setting_use_blurhash: Show colorful gradients for hidden media
         setting_use_pending_items: Slow mode
diff --git a/config/settings.yml b/config/settings.yml
index 10180201f..4e5eefb59 100644
--- a/config/settings.yml
+++ b/config/settings.yml
@@ -34,6 +34,7 @@ defaults: &defaults
   advanced_layout: false
   use_blurhash: true
   use_pending_items: false
+  trends: true
   notification_emails:
     follow: false
     reblog: false
-- 
cgit 


From a8958d07e222f2aa6867e8a5bf93cce4535fd3ce Mon Sep 17 00:00:00 2001
From: Eugen Rochko <eugen@zeonfederated.com>
Date: Wed, 7 Aug 2019 10:00:58 +0200
Subject: Fix featured tags not finding the right tag on save (#11504)

Regression from f371b32
---
 app/javascript/styles/mastodon/widgets.scss | 7 ++++++-
 app/models/featured_tag.rb                  | 2 +-
 2 files changed, 7 insertions(+), 2 deletions(-)

(limited to 'app/models')

diff --git a/app/javascript/styles/mastodon/widgets.scss b/app/javascript/styles/mastodon/widgets.scss
index acaf5b024..8c30bc57c 100644
--- a/app/javascript/styles/mastodon/widgets.scss
+++ b/app/javascript/styles/mastodon/widgets.scss
@@ -324,7 +324,8 @@
     &.active h4 {
       &,
       .fa,
-      small {
+      small,
+      .trends__item__current {
         color: $primary-text-color;
       }
     }
@@ -337,6 +338,10 @@
     &.active .avatar-stack .account__avatar {
       border-color: $ui-highlight-color;
     }
+
+    .trends__item__current {
+      padding-right: 0;
+    }
   }
 }
 
diff --git a/app/models/featured_tag.rb b/app/models/featured_tag.rb
index d06ae26a8..e02ae0705 100644
--- a/app/models/featured_tag.rb
+++ b/app/models/featured_tag.rb
@@ -23,7 +23,7 @@ class FeaturedTag < ApplicationRecord
   validate :validate_featured_tags_limit, on: :create
 
   def name=(str)
-    self.tag = Tag.find_or_initialize_by(name: str.strip.delete('#').mb_chars.downcase.to_s)
+    self.tag = Tag.find_or_create_by_names(str.strip)&.first
   end
 
   def increment(timestamp)
-- 
cgit 


From ac33f1aedd9a6c72c6c176afb1f5d62a1ce5d44d Mon Sep 17 00:00:00 2001
From: Eugen Rochko <eugen@zeonfederated.com>
Date: Wed, 7 Aug 2019 10:01:55 +0200
Subject: Fix account tags not being saved correctly (#11507)

* Fix account tags not being saved correctly

Regression from f371b32

Fix Tag#discoverable not returning tags where listable is nil instead of true

Add notice when saving hashtags in admin UI

Change public hashtag and directory pages to return 404 for forbidden tags

* Remove unused locale string
---
 app/controllers/admin/tags_controller.rb  |  2 +-
 app/controllers/directories_controller.rb |  2 +-
 app/controllers/tags_controller.rb        |  2 +-
 app/models/account.rb                     | 12 +-----------
 app/models/tag.rb                         |  3 ++-
 config/locales/en.yml                     |  1 +
 6 files changed, 7 insertions(+), 15 deletions(-)

(limited to 'app/models')

diff --git a/app/controllers/admin/tags_controller.rb b/app/controllers/admin/tags_controller.rb
index 0e9dda302..ed271aedc 100644
--- a/app/controllers/admin/tags_controller.rb
+++ b/app/controllers/admin/tags_controller.rb
@@ -17,7 +17,7 @@ module Admin
       authorize @tag, :update?
 
       if @tag.update(tag_params.merge(reviewed_at: Time.now.utc))
-        redirect_to admin_tag_path(@tag.id)
+        redirect_to admin_tag_path(@tag.id), notice: I18n.t('admin.tags.updated_msg')
       else
         render :show
       end
diff --git a/app/controllers/directories_controller.rb b/app/controllers/directories_controller.rb
index d2ef76f06..a5c47b515 100644
--- a/app/controllers/directories_controller.rb
+++ b/app/controllers/directories_controller.rb
@@ -25,7 +25,7 @@ class DirectoriesController < ApplicationController
   end
 
   def set_tag
-    @tag = Tag.discoverable.find_by!(name: params[:id].downcase)
+    @tag = Tag.discoverable.find_normalized!(params[:id])
   end
 
   def set_tags
diff --git a/app/controllers/tags_controller.rb b/app/controllers/tags_controller.rb
index 3cd2d9e20..5a6fcc8fd 100644
--- a/app/controllers/tags_controller.rb
+++ b/app/controllers/tags_controller.rb
@@ -47,7 +47,7 @@ class TagsController < ApplicationController
   private
 
   def set_tag
-    @tag = Tag.find_normalized!(params[:id])
+    @tag = Tag.usable.find_normalized!(params[:id])
   end
 
   def set_body_classes
diff --git a/app/models/account.rb b/app/models/account.rb
index ccd116d6e..b205c8c9e 100644
--- a/app/models/account.rb
+++ b/app/models/account.rb
@@ -227,17 +227,7 @@ class Account < ApplicationRecord
   end
 
   def tags_as_strings=(tag_names)
-    tag_names.map! { |name| name.mb_chars.downcase.to_s }
-    tag_names.uniq!
-
-    # Existing hashtags
-    hashtags_map = Tag.where(name: tag_names).each_with_object({}) { |tag, h| h[tag.name] = tag }
-
-    # Initialize not yet existing hashtags
-    tag_names.each do |name|
-      next if hashtags_map.key?(name)
-      hashtags_map[name] = Tag.new(name: name)
-    end
+    hashtags_map = Tag.find_or_create_by_names(tag_names).each_with_object({}) { |tag, h| h[tag.name] = tag }
 
     # Remove hashtags that are to be deleted
     tags.each do |tag|
diff --git a/app/models/tag.rb b/app/models/tag.rb
index 6a02581fa..e2fe91da1 100644
--- a/app/models/tag.rb
+++ b/app/models/tag.rb
@@ -31,7 +31,8 @@ class Tag < ApplicationRecord
 
   scope :reviewed, -> { where.not(reviewed_at: nil) }
   scope :pending_review, -> { where(reviewed_at: nil).where.not(requested_review_at: nil) }
-  scope :discoverable, -> { where.not(listable: false).joins(:account_tag_stat).where(AccountTagStat.arel_table[:accounts_count].gt(0)).order(Arel.sql('account_tag_stats.accounts_count desc')) }
+  scope :usable, -> { where(usable: [true, nil]) }
+  scope :discoverable, -> { where(listable: [true, nil]).joins(:account_tag_stat).where(AccountTagStat.arel_table[:accounts_count].gt(0)).order(Arel.sql('account_tag_stats.accounts_count desc')) }
   scope :most_used, ->(account) { joins(:statuses).where(statuses: { account: account }).group(:id).order(Arel.sql('count(*) desc')) }
 
   delegate :accounts_count,
diff --git a/config/locales/en.yml b/config/locales/en.yml
index 333d4f172..20baf634e 100644
--- a/config/locales/en.yml
+++ b/config/locales/en.yml
@@ -496,6 +496,7 @@ en:
       title: Hashtags
       trending_right_now: Trending right now
       unique_uses_today: "%{count} posting today"
+      updated_msg: Hashtag settings updated successfully
     title: Administration
     warning_presets:
       add_new: Add new
-- 
cgit 


From 94c54997cf6dc3bef2af67a070a61cc10595339c Mon Sep 17 00:00:00 2001
From: Eugen Rochko <eugen@zeonfederated.com>
Date: Wed, 7 Aug 2019 17:08:30 +0200
Subject: Fix trending tags returning less items than requested sometimes
 (#11513)

Add better sorting defaults to the hashtags admin UI

Add "not reviewed" filter to hashtags admin UI
---
 app/controllers/admin/tags_controller.rb | 7 ++++---
 app/models/tag.rb                        | 3 ++-
 app/models/trending_tags.rb              | 9 +++++----
 app/views/admin/tags/index.html.haml     | 1 +
 config/locales/en.yml                    | 1 +
 5 files changed, 13 insertions(+), 8 deletions(-)

(limited to 'app/models')

diff --git a/app/controllers/admin/tags_controller.rb b/app/controllers/admin/tags_controller.rb
index ed271aedc..794bb114a 100644
--- a/app/controllers/admin/tags_controller.rb
+++ b/app/controllers/admin/tags_controller.rb
@@ -36,9 +36,10 @@ module Admin
     def filtered_tags
       scope = Tag
       scope = scope.discoverable if filter_params[:context] == 'directory'
-      scope = scope.reviewed if filter_params[:review] == 'reviewed'
-      scope = scope.pending_review if filter_params[:review] == 'pending_review'
-      scope.reorder(score: :desc)
+      scope = scope.unreviewed if filter_params[:review] == 'unreviewed'
+      scope = scope.reviewed.order(reviewed_at: :desc) if filter_params[:review] == 'reviewed'
+      scope = scope.pending_review.order(requested_review_at: :desc) if filter_params[:review] == 'pending_review'
+      scope.order(score: :desc)
     end
 
     def filter_params
diff --git a/app/models/tag.rb b/app/models/tag.rb
index e2fe91da1..1364d1dba 100644
--- a/app/models/tag.rb
+++ b/app/models/tag.rb
@@ -30,7 +30,8 @@ class Tag < ApplicationRecord
   validate :validate_name_change, if: -> { !new_record? && name_changed? }
 
   scope :reviewed, -> { where.not(reviewed_at: nil) }
-  scope :pending_review, -> { where(reviewed_at: nil).where.not(requested_review_at: nil) }
+  scope :unreviewed, -> { where(reviewed_at: nil) }
+  scope :pending_review, -> { unreviewed.where.not(requested_review_at: nil) }
   scope :usable, -> { where(usable: [true, nil]) }
   scope :discoverable, -> { where(listable: [true, nil]).joins(:account_tag_stat).where(AccountTagStat.arel_table[:accounts_count].gt(0)).order(Arel.sql('account_tag_stats.accounts_count desc')) }
   scope :most_used, ->(account) { joins(:statuses).where(statuses: { account: account }).group(:id).order(Arel.sql('count(*) desc')) }
diff --git a/app/models/trending_tags.rb b/app/models/trending_tags.rb
index 0a7e2feac..594ae9520 100644
--- a/app/models/trending_tags.rb
+++ b/app/models/trending_tags.rb
@@ -5,6 +5,7 @@ class TrendingTags
   EXPIRE_HISTORY_AFTER = 7.days.seconds
   EXPIRE_TRENDS_AFTER  = 1.day.seconds
   THRESHOLD            = 5
+  LIMIT                = 10
 
   class << self
     include Redisable
@@ -18,18 +19,18 @@ class TrendingTags
     end
 
     def get(limit, filtered: true)
-      tag_ids = redis.zrevrange("#{KEY}:#{Time.now.utc.beginning_of_day.to_i}", 0, limit - 1).map(&:to_i)
+      tag_ids = redis.zrevrange("#{KEY}:#{Time.now.utc.beginning_of_day.to_i}", 0, LIMIT - 1).map(&:to_i)
 
       tags = Tag.where(id: tag_ids)
       tags = tags.where(trendable: true) if filtered
       tags = tags.each_with_object({}) { |tag, h| h[tag.id] = tag }
 
-      tag_ids.map { |tag_id| tags[tag_id] }.compact
+      tag_ids.map { |tag_id| tags[tag_id] }.compact.take(limit)
     end
 
     def trending?(tag)
       rank = redis.zrevrank("#{KEY}:#{Time.now.utc.beginning_of_day.to_i}", tag.id)
-      rank.present? && rank <= 10
+      rank.present? && rank <= LIMIT
     end
 
     private
@@ -59,7 +60,7 @@ class TrendingTags
         old_rank = redis.zrevrank(key, tag.id)
 
         redis.zadd(key, score, tag.id)
-        request_review!(tag) if (old_rank.nil? || old_rank > 10) && redis.zrevrank(key, tag.id) <= 10 && !tag.trendable? && tag.requires_review? && !tag.requested_review?
+        request_review!(tag) if (old_rank.nil? || old_rank > LIMIT) && redis.zrevrank(key, tag.id) <= LIMIT && !tag.trendable? && tag.requires_review? && !tag.requested_review?
       end
 
       redis.expire(key, EXPIRE_TRENDS_AFTER)
diff --git a/app/views/admin/tags/index.html.haml b/app/views/admin/tags/index.html.haml
index 5e4ee21f5..d994955ef 100644
--- a/app/views/admin/tags/index.html.haml
+++ b/app/views/admin/tags/index.html.haml
@@ -12,6 +12,7 @@
     %strong= t('admin.tags.review')
     %ul
       %li= filter_link_to t('generic.all'), review: nil
+      %li= filter_link_to t('admin.tags.unreviewed'), review: 'unreviewed'
       %li= filter_link_to t('admin.tags.reviewed'), review: 'reviewed'
       %li= filter_link_to safe_join([t('admin.accounts.moderation.pending'), "(#{Tag.pending_review.count})"], ' '), review: 'pending_review'
 
diff --git a/config/locales/en.yml b/config/locales/en.yml
index 7b24df016..17ff24726 100644
--- a/config/locales/en.yml
+++ b/config/locales/en.yml
@@ -498,6 +498,7 @@ en:
       title: Hashtags
       trending_right_now: Trending right now
       unique_uses_today: "%{count} posting today"
+      unreviewed: Not reviewed
       updated_msg: Hashtag settings updated successfully
     title: Administration
     warning_presets:
-- 
cgit 


From bced70469a6c4aecdb3c71055f329a0f579eb14c Mon Sep 17 00:00:00 2001
From: ThibG <thib@sitedethib.com>
Date: Wed, 7 Aug 2019 20:20:23 +0200
Subject: Add domain block notes (#11515)
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

* Add database columns for adding notes to domain blocks/restrctions

* Add admin UI to set private and public comments when blocking a domain

* Add text for private and public comments on domain blocks

* Show domain block comments in admin UI

* Add comments to the domain block undo page

* Make UnblockDomainService more robust regarding upgraded domain blocks

* Allow editing domain blocks

* Rename button from “undo domain block” to “view domain block” in account admin UI

* Change test to unsilence silenced users from upgraded blocks
---
 app/controllers/admin/domain_blocks_controller.rb  | 28 ++++++++++++++++++--
 app/controllers/admin/instances_controller.rb      |  2 ++
 app/models/domain_block.rb                         | 16 +++++++-----
 app/services/block_domain_service.rb               | 11 +++++++-
 app/services/unblock_domain_service.rb             | 19 ++------------
 app/views/admin/accounts/show.html.haml            |  2 +-
 app/views/admin/domain_blocks/edit.html.haml       | 30 ++++++++++++++++++++++
 app/views/admin/domain_blocks/new.html.haml        |  6 +++++
 app/views/admin/domain_blocks/show.html.haml       | 12 +++++++++
 app/views/admin/instances/show.html.haml           | 13 ++++++++++
 app/workers/domain_block_worker.rb                 |  4 +--
 config/locales/en.yml                              |  8 ++++++
 config/routes.rb                                   |  6 ++++-
 ...20190807135426_add_comments_to_domain_blocks.rb |  7 +++++
 db/schema.rb                                       |  4 ++-
 spec/services/unblock_domain_service_spec.rb       |  2 +-
 spec/workers/domain_block_worker_spec.rb           |  2 +-
 17 files changed, 138 insertions(+), 34 deletions(-)
 create mode 100644 app/views/admin/domain_blocks/edit.html.haml
 create mode 100644 db/migrate/20190807135426_add_comments_to_domain_blocks.rb

(limited to 'app/models')

diff --git a/app/controllers/admin/domain_blocks_controller.rb b/app/controllers/admin/domain_blocks_controller.rb
index 7129656da..74a36b79c 100644
--- a/app/controllers/admin/domain_blocks_controller.rb
+++ b/app/controllers/admin/domain_blocks_controller.rb
@@ -2,13 +2,17 @@
 
 module Admin
   class DomainBlocksController < BaseController
-    before_action :set_domain_block, only: [:show, :destroy]
+    before_action :set_domain_block, only: [:show, :destroy, :edit, :update]
 
     def new
       authorize :domain_block, :create?
       @domain_block = DomainBlock.new(domain: params[:_domain])
     end
 
+    def edit
+      authorize :domain_block, :create?
+    end
+
     def create
       authorize :domain_block, :create?
 
@@ -35,6 +39,22 @@ module Admin
       end
     end
 
+    def update
+      authorize :domain_block, :create?
+
+      @domain_block.update(update_params)
+
+      severity_changed = @domain_block.severity_changed?
+
+      if @domain_block.save
+        DomainBlockWorker.perform_async(@domain_block.id, severity_changed)
+        log_action :create, @domain_block
+        redirect_to admin_instances_path(limited: '1'), notice: I18n.t('admin.domain_blocks.created_msg')
+      else
+        render :edit
+      end
+    end
+
     def show
       authorize @domain_block, :show?
     end
@@ -52,8 +72,12 @@ module Admin
       @domain_block = DomainBlock.find(params[:id])
     end
 
+    def update_params
+      params.require(:domain_block).permit(:severity, :reject_media, :reject_reports, :private_comment, :public_comment)
+    end
+
     def resource_params
-      params.require(:domain_block).permit(:domain, :severity, :reject_media, :reject_reports)
+      params.require(:domain_block).permit(:domain, :severity, :reject_media, :reject_reports, :private_comment, :public_comment)
     end
   end
 end
diff --git a/app/controllers/admin/instances_controller.rb b/app/controllers/admin/instances_controller.rb
index d4f201807..b47b18f8e 100644
--- a/app/controllers/admin/instances_controller.rb
+++ b/app/controllers/admin/instances_controller.rb
@@ -21,6 +21,8 @@ module Admin
       @blocks_count    = Block.where(target_account: Account.where(domain: params[:id])).count
       @available       = DeliveryFailureTracker.available?(Account.select(:shared_inbox_url).where(domain: params[:id]).first&.shared_inbox_url)
       @media_storage   = MediaAttachment.where(account: Account.where(domain: params[:id])).sum(:file_file_size)
+      @private_comment = @domain_block&.private_comment
+      @public_comment  = @domain_block&.public_comment
     end
 
     private
diff --git a/app/models/domain_block.rb b/app/models/domain_block.rb
index 25d3b87ef..3f5b9f23e 100644
--- a/app/models/domain_block.rb
+++ b/app/models/domain_block.rb
@@ -3,13 +3,15 @@
 #
 # Table name: domain_blocks
 #
-#  id             :bigint(8)        not null, primary key
-#  domain         :string           default(""), not null
-#  created_at     :datetime         not null
-#  updated_at     :datetime         not null
-#  severity       :integer          default("silence")
-#  reject_media   :boolean          default(FALSE), not null
-#  reject_reports :boolean          default(FALSE), not null
+#  id              :bigint(8)        not null, primary key
+#  domain          :string           default(""), not null
+#  created_at      :datetime         not null
+#  updated_at      :datetime         not null
+#  severity        :integer          default("silence")
+#  reject_media    :boolean          default(FALSE), not null
+#  reject_reports  :boolean          default(FALSE), not null
+#  private_comment :text
+#  public_comment  :text
 #
 
 class DomainBlock < ApplicationRecord
diff --git a/app/services/block_domain_service.rb b/app/services/block_domain_service.rb
index c5e5e5761..0ec6be503 100644
--- a/app/services/block_domain_service.rb
+++ b/app/services/block_domain_service.rb
@@ -3,13 +3,22 @@
 class BlockDomainService < BaseService
   attr_reader :domain_block
 
-  def call(domain_block)
+  def call(domain_block, update = false)
     @domain_block = domain_block
     process_domain_block!
+    process_retroactive_updates! if update
   end
 
   private
 
+  def process_retroactive_updates!
+    # If the domain block severity has been changed, undo the appropriate limitations
+    scope = Account.by_domain_and_subdomains(domain_block.domain)
+
+    scope.where(silenced_at: domain_block.created_at).in_batches.update_all(silenced_at: nil) unless domain_block.silence?
+    scope.where(suspended_at: domain_block.created_at).in_batches.update_all(suspended_at: nil) unless domain_block.suspend?
+  end
+
   def process_domain_block!
     clear_media! if domain_block.reject_media?
 
diff --git a/app/services/unblock_domain_service.rb b/app/services/unblock_domain_service.rb
index fc262a50a..d502d9e49 100644
--- a/app/services/unblock_domain_service.rb
+++ b/app/services/unblock_domain_service.rb
@@ -10,24 +10,9 @@ class UnblockDomainService < BaseService
   end
 
   def process_retroactive_updates
-    blocked_accounts.in_batches.update_all(update_options) unless domain_block.noop?
-  end
-
-  def blocked_accounts
     scope = Account.by_domain_and_subdomains(domain_block.domain)
 
-    if domain_block.silence?
-      scope.where(silenced_at: @domain_block.created_at)
-    else
-      scope.where(suspended_at: @domain_block.created_at)
-    end
-  end
-
-  def update_options
-    { domain_block_impact => nil }
-  end
-
-  def domain_block_impact
-    domain_block.silence? ? :silenced_at : :suspended_at
+    scope.where(silenced_at: domain_block.created_at).in_batches.update_all(silenced_at: nil) unless domain_block.noop?
+    scope.where(suspended_at: domain_block.created_at).in_batches.update_all(suspended_at: nil) if domain_block.suspend?
   end
 end
diff --git a/app/views/admin/accounts/show.html.haml b/app/views/admin/accounts/show.html.haml
index 7494c9fa2..59babd3b0 100644
--- a/app/views/admin/accounts/show.html.haml
+++ b/app/views/admin/accounts/show.html.haml
@@ -174,7 +174,7 @@
 
       - unless @account.local?
         - if DomainBlock.where(domain: @account.domain).exists?
-          = link_to t('admin.domain_blocks.undo'), admin_instance_path(@account.domain), class: 'button'
+          = link_to t('admin.domain_blocks.view'), admin_instance_path(@account.domain), class: 'button'
         - else
           = link_to t('admin.domain_blocks.add_new'), new_admin_domain_block_path(_domain: @account.domain), class: 'button button--destructive'
 
diff --git a/app/views/admin/domain_blocks/edit.html.haml b/app/views/admin/domain_blocks/edit.html.haml
new file mode 100644
index 000000000..29e47ef3b
--- /dev/null
+++ b/app/views/admin/domain_blocks/edit.html.haml
@@ -0,0 +1,30 @@
+- content_for :header_tags do
+  = javascript_pack_tag 'admin', integrity: true, async: true, crossorigin: 'anonymous'
+
+- content_for :page_title do
+  = t('admin.domain_blocks.edit')
+
+= simple_form_for @domain_block, url: admin_domain_block_path(@domain_block), method: :put do |f|
+  = render 'shared/error_messages', object: @domain_block
+
+  .fields-row
+    .fields-row__column.fields-row__column-6.fields-group
+      = f.input :domain, wrapper: :with_label, label: t('admin.domain_blocks.domain'), hint: t('admin.domain_blocks.new.hint'), required: true, readonly: true, disabled: true
+
+    .fields-row__column.fields-row__column-6.fields-group
+      = f.input :severity, collection: DomainBlock.severities.keys, wrapper: :with_label, include_blank: false, label_method: lambda { |type| t("admin.domain_blocks.new.severity.#{type}") }, hint: t('admin.domain_blocks.new.severity.desc_html')
+
+  .fields-group
+    = f.input :reject_media, as: :boolean, wrapper: :with_label, label: I18n.t('admin.domain_blocks.reject_media'), hint: I18n.t('admin.domain_blocks.reject_media_hint')
+
+  .fields-group
+    = f.input :reject_reports, as: :boolean, wrapper: :with_label, label: I18n.t('admin.domain_blocks.reject_reports'), hint: I18n.t('admin.domain_blocks.reject_reports_hint')
+
+  .field-group
+    = f.input :private_comment, wrapper: :with_label, label: I18n.t('admin.domain_blocks.private_comment'), hint: t('admin.domain_blocks.private_comment_hint'), rows: 6
+
+  .field-group
+    = f.input :public_comment, wrapper: :with_label, label: I18n.t('admin.domain_blocks.public_comment'), hint: t('admin.domain_blocks.public_comment_hint'), rows: 6
+
+  .actions
+    = f.button :button, t('generic.save_changes'), type: :submit
diff --git a/app/views/admin/domain_blocks/new.html.haml b/app/views/admin/domain_blocks/new.html.haml
index 055d2fbd7..ed1581936 100644
--- a/app/views/admin/domain_blocks/new.html.haml
+++ b/app/views/admin/domain_blocks/new.html.haml
@@ -20,5 +20,11 @@
   .fields-group
     = f.input :reject_reports, as: :boolean, wrapper: :with_label, label: I18n.t('admin.domain_blocks.reject_reports'), hint: I18n.t('admin.domain_blocks.reject_reports_hint')
 
+  .field-group
+    = f.input :private_comment, wrapper: :with_label, label: I18n.t('admin.domain_blocks.private_comment'), hint: t('admin.domain_blocks.private_comment_hint'), rows: 6
+
+  .field-group
+    = f.input :public_comment, wrapper: :with_label, label: I18n.t('admin.domain_blocks.public_comment'), hint: t('admin.domain_blocks.public_comment_hint'), rows: 6
+
   .actions
     = f.button :button, t('.create'), type: :submit
diff --git a/app/views/admin/domain_blocks/show.html.haml b/app/views/admin/domain_blocks/show.html.haml
index dca4dbac7..e64aaa629 100644
--- a/app/views/admin/domain_blocks/show.html.haml
+++ b/app/views/admin/domain_blocks/show.html.haml
@@ -1,6 +1,18 @@
 - content_for :page_title do
   = t('admin.domain_blocks.show.title', domain: @domain_block.domain)
 
+- if @domain_block.private_comment.present?
+  .speech-bubble
+    .speech-bubble__bubble
+      = simple_format(h(@domain_block.private_comment))
+    .speech-bubble__owner= t 'admin.instances.private_comment'
+
+- if @domain_block.public_comment.present?
+  .speech-bubble
+    .speech-bubble__bubble
+      = simple_format(h(@domain_block.public_comment))
+    .speech-bubble__owner= t 'admin.instances.public_comment'
+
 = simple_form_for @domain_block, url: admin_domain_block_path(@domain_block), method: :delete do |f|
 
   - unless (@domain_block.noop?)
diff --git a/app/views/admin/instances/show.html.haml b/app/views/admin/instances/show.html.haml
index fbb49ba02..294c9495d 100644
--- a/app/views/admin/instances/show.html.haml
+++ b/app/views/admin/instances/show.html.haml
@@ -31,6 +31,18 @@
           = fa_icon 'times'
       .dashboard__counters__label= t 'admin.instances.delivery_available'
 
+- if @private_comment.present?
+  .speech-bubble
+    .speech-bubble__bubble
+      = simple_format(h(@private_comment))
+    .speech-bubble__owner= t 'admin.instances.private_comment'
+
+- if @public_comment.present?
+  .speech-bubble
+    .speech-bubble__bubble
+      = simple_format(h(@public_comment))
+    .speech-bubble__owner= t 'admin.instances.public_comment'
+
 %hr.spacer/
 
 %div{ style: 'overflow: hidden' }
@@ -41,6 +53,7 @@
     - if @domain_allow
       = link_to t('admin.domain_allows.undo'), admin_domain_allow_path(@domain_allow), class: 'button button--destructive', data: { confirm: t('admin.accounts.are_you_sure'), method: :delete }
     - elsif @domain_block
+      = link_to t('admin.domain_blocks.edit'), edit_admin_domain_block_path(@domain_block), class: 'button'
       = link_to t('admin.domain_blocks.undo'), admin_domain_block_path(@domain_block), class: 'button'
     - else
       = link_to t('admin.domain_blocks.add_new'), new_admin_domain_block_path(_domain: @instance.domain), class: 'button'
diff --git a/app/workers/domain_block_worker.rb b/app/workers/domain_block_worker.rb
index 884477829..35518d6b5 100644
--- a/app/workers/domain_block_worker.rb
+++ b/app/workers/domain_block_worker.rb
@@ -3,8 +3,8 @@
 class DomainBlockWorker
   include Sidekiq::Worker
 
-  def perform(domain_block_id)
-    BlockDomainService.new.call(DomainBlock.find(domain_block_id))
+  def perform(domain_block_id, update = false)
+    BlockDomainService.new.call(DomainBlock.find(domain_block_id), update)
   rescue ActiveRecord::RecordNotFound
     true
   end
diff --git a/config/locales/en.yml b/config/locales/en.yml
index 17ff24726..b677a6651 100644
--- a/config/locales/en.yml
+++ b/config/locales/en.yml
@@ -284,6 +284,7 @@ en:
       created_msg: Domain block is now being processed
       destroyed_msg: Domain block has been undone
       domain: Domain
+      edit: Edit domain block
       existing_domain_block_html: You have already imposed stricter limits on %{name}, you need to <a href="%{unblock_url}">unblock it</a> first.
       new:
         create: Create block
@@ -294,6 +295,10 @@ en:
           silence: Silence
           suspend: Suspend
         title: New domain block
+      private_comment: Private comment
+      private_comment_hint: Comment about this domain limitation for internal use by the moderators.
+      public_comment: Public comment
+      public_comment_hint: Comment about this domain limitation for the general public, if advertising the list of domain limitations is enabled.
       reject_media: Reject media files
       reject_media_hint: Removes locally stored media files and refuses to download any in the future. Irrelevant for suspensions
       reject_reports: Reject reports
@@ -313,6 +318,7 @@ en:
         title: Undo domain block for %{domain}
         undo: Undo
       undo: Undo domain block
+      view: View domain block
     email_domain_blocks:
       add_new: Add new
       created_msg: Successfully added e-mail domain to blacklist
@@ -336,6 +342,8 @@ en:
         all: All
         limited: Limited
         title: Moderation
+      private_comment: Private comment
+      public_comment: Public comment
       title: Federation
       total_blocked_by_us: Blocked by us
       total_followed_by_them: Followed by them
diff --git a/config/routes.rb b/config/routes.rb
index 60f7d2e05..9c33b8190 100644
--- a/config/routes.rb
+++ b/config/routes.rb
@@ -155,7 +155,11 @@ Rails.application.routes.draw do
     get '/dashboard', to: 'dashboard#index'
 
     resources :domain_allows, only: [:new, :create, :show, :destroy]
-    resources :domain_blocks, only: [:new, :create, :show, :destroy]
+    resources :domain_blocks, only: [:new, :create, :show, :destroy, :update] do
+      member do
+        get :edit
+      end
+    end
     resources :email_domain_blocks, only: [:index, :new, :create, :destroy]
     resources :action_logs, only: [:index]
     resources :warning_presets, except: [:new]
diff --git a/db/migrate/20190807135426_add_comments_to_domain_blocks.rb b/db/migrate/20190807135426_add_comments_to_domain_blocks.rb
new file mode 100644
index 000000000..b660a71ad
--- /dev/null
+++ b/db/migrate/20190807135426_add_comments_to_domain_blocks.rb
@@ -0,0 +1,7 @@
+class AddCommentsToDomainBlocks < ActiveRecord::Migration[5.2]
+  def change
+    add_column :domain_blocks, :private_comment, :text
+    add_column :domain_blocks, :public_comment, :text
+  end
+end
+
diff --git a/db/schema.rb b/db/schema.rb
index d1b6825b4..f8fc6a821 100644
--- a/db/schema.rb
+++ b/db/schema.rb
@@ -10,7 +10,7 @@
 #
 # It's strongly recommended that you check this file into your version control system.
 
-ActiveRecord::Schema.define(version: 2019_08_05_123746) do
+ActiveRecord::Schema.define(version: 2019_08_07_135426) do
 
   # These are extensions that must be enabled in order to support this database
   enable_extension "plpgsql"
@@ -259,6 +259,8 @@ ActiveRecord::Schema.define(version: 2019_08_05_123746) do
     t.integer "severity", default: 0
     t.boolean "reject_media", default: false, null: false
     t.boolean "reject_reports", default: false, null: false
+    t.text "private_comment"
+    t.text "public_comment"
     t.index ["domain"], name: "index_domain_blocks_on_domain", unique: true
   end
 
diff --git a/spec/services/unblock_domain_service_spec.rb b/spec/services/unblock_domain_service_spec.rb
index 619aefb5c..27dbc92ad 100644
--- a/spec/services/unblock_domain_service_spec.rb
+++ b/spec/services/unblock_domain_service_spec.rb
@@ -31,7 +31,7 @@ describe UnblockDomainService, type: :service do
       subject.call(@domain_block)
       expect_deleted_domain_block
       expect(@suspended.reload.suspended?).to be false
-      expect(@silenced.reload.silenced?).to be true
+      expect(@silenced.reload.silenced?).to be false
       expect(@independently_suspended.reload.suspended?).to be true
       expect(@independently_silenced.reload.silenced?).to be true
     end
diff --git a/spec/workers/domain_block_worker_spec.rb b/spec/workers/domain_block_worker_spec.rb
index c4138501f..48b3e38c4 100644
--- a/spec/workers/domain_block_worker_spec.rb
+++ b/spec/workers/domain_block_worker_spec.rb
@@ -14,7 +14,7 @@ describe DomainBlockWorker do
       result = subject.perform(domain_block.id)
 
       expect(result).to be_nil
-      expect(service).to have_received(:call).with(domain_block)
+      expect(service).to have_received(:call).with(domain_block, false)
     end
 
     it 'calls domain block service for relevant domain block' do
-- 
cgit 


From f51c7c105f1d04520656c1235f8a5f58d256fd0e Mon Sep 17 00:00:00 2001
From: Eugen Rochko <eugen@zeonfederated.com>
Date: Wed, 7 Aug 2019 21:14:08 +0200
Subject: Fix acct URIs with IDN domains not being resolved (#11520)

Fix #11494
---
 app/controllers/remote_interaction_controller.rb |  1 -
 app/models/remote_follow.rb                      | 36 ++++++++++++++++++++----
 app/services/resolve_account_service.rb          | 14 ++++++---
 3 files changed, 40 insertions(+), 11 deletions(-)

(limited to 'app/models')

diff --git a/app/controllers/remote_interaction_controller.rb b/app/controllers/remote_interaction_controller.rb
index fa742fb0a..de5616e25 100644
--- a/app/controllers/remote_interaction_controller.rb
+++ b/app/controllers/remote_interaction_controller.rb
@@ -39,7 +39,6 @@ class RemoteInteractionController < ApplicationController
     @status = Status.find(params[:id])
     authorize @status, :show?
   rescue Mastodon::NotPermittedError
-    # Reraise in order to get a 404
     raise ActiveRecord::RecordNotFound
   end
 
diff --git a/app/models/remote_follow.rb b/app/models/remote_follow.rb
index 2537de36c..93df11724 100644
--- a/app/models/remote_follow.rb
+++ b/app/models/remote_follow.rb
@@ -2,19 +2,21 @@
 
 class RemoteFollow
   include ActiveModel::Validations
+  include RoutingHelper
 
   attr_accessor :acct, :addressable_template
 
   validates :acct, presence: true
 
-  def initialize(attrs = nil)
-    @acct = attrs[:acct].gsub(/\A@/, '').strip if !attrs.nil? && !attrs[:acct].nil?
+  def initialize(attrs = {})
+    @acct = normalize_acct(attrs[:acct])
   end
 
   def valid?
     return false unless super
 
-    populate_template
+    fetch_template!
+
     errors.empty?
   end
 
@@ -28,8 +30,30 @@ class RemoteFollow
 
   private
 
-  def populate_template
-    if acct.blank? || redirect_url_link.nil? || redirect_url_link.template.nil?
+  def normalize_acct(value)
+    return if value.blank?
+
+    username, domain = value.strip.gsub(/\A@/, '').split('@')
+
+    domain = begin
+      if TagManager.instance.local_domain?(domain)
+        nil
+      else
+        TagManager.instance.normalize_domain(domain)
+      end
+    end
+
+    [username, domain].compact.join('@')
+  end
+
+  def fetch_template!
+    return missing_resource if acct.blank?
+
+    _, domain = acct.split('@')
+
+    if domain.nil?
+      @addressable_template = Addressable::Template.new("#{authorize_interaction_url}?uri={uri}")
+    elsif redirect_url_link.nil? || redirect_url_link.template.nil?
       missing_resource_error
     else
       @addressable_template = Addressable::Template.new(redirect_uri_template)
@@ -45,7 +69,7 @@ class RemoteFollow
   end
 
   def acct_resource
-    @_acct_resource ||= Goldfinger.finger("acct:#{acct}")
+    @acct_resource ||= Goldfinger.finger("acct:#{acct}")
   rescue Goldfinger::Error, HTTP::ConnectionError
     nil
   end
diff --git a/app/services/resolve_account_service.rb b/app/services/resolve_account_service.rb
index 7864c4bcd..12e6544a0 100644
--- a/app/services/resolve_account_service.rb
+++ b/app/services/resolve_account_service.rb
@@ -60,17 +60,23 @@ class ResolveAccountService < BaseService
       @account  = uri
       @username = @account.username
       @domain   = @account.domain
-      @uri      = [@username, @domain].compact.join('@')
     else
-      @uri               = uri
       @username, @domain = uri.split('@')
     end
 
-    @domain = nil if TagManager.instance.local_domain?(@domain)
+    @domain = begin
+      if TagManager.instance.local_domain?(@domain)
+        nil
+      else
+        TagManager.instance.normalize_domain(@domain)
+      end
+    end
+
+    @uri = [@username, @domain].compact.join('@')
   end
 
   def process_webfinger!(uri, redirected = false)
-    @webfinger                           = Goldfinger.finger("acct:#{@uri}")
+    @webfinger                           = Goldfinger.finger("acct:#{uri}")
     confirmed_username, confirmed_domain = @webfinger.subject.gsub(/\Aacct:/, '').split('@')
 
     if confirmed_username.casecmp(@username).zero? && confirmed_domain.casecmp(@domain).zero?
-- 
cgit 


From 41aac1ba3a247b2178699e529239a7f3b8eb6edc Mon Sep 17 00:00:00 2001
From: Eugen Rochko <eugen@zeonfederated.com>
Date: Thu, 8 Aug 2019 23:03:09 +0200
Subject: Add GIF and WebP support for custom emojis (#11519)

Fix #11466
---
 app/models/custom_emoji.rb | 4 +++-
 1 file changed, 3 insertions(+), 1 deletion(-)

(limited to 'app/models')

diff --git a/app/models/custom_emoji.rb b/app/models/custom_emoji.rb
index 514cf4845..643a7e46a 100644
--- a/app/models/custom_emoji.rb
+++ b/app/models/custom_emoji.rb
@@ -28,6 +28,8 @@ class CustomEmoji < ApplicationRecord
     :(#{SHORTCODE_RE_FRAGMENT}):
     (?=[^[:alnum:]:]|$)/x
 
+  IMAGE_MIME_TYPES = %w(image/png image/gif image/webp).freeze
+
   belongs_to :category, class_name: 'CustomEmojiCategory', optional: true
   has_one :local_counterpart, -> { where(domain: nil) }, class_name: 'CustomEmoji', primary_key: :shortcode, foreign_key: :shortcode
 
@@ -35,7 +37,7 @@ class CustomEmoji < ApplicationRecord
 
   before_validation :downcase_domain
 
-  validates_attachment :image, content_type: { content_type: 'image/png' }, presence: true, size: { less_than: LIMIT }
+  validates_attachment :image, content_type: { content_type: IMAGE_MIME_TYPES }, presence: true, size: { less_than: LIMIT }
   validates :shortcode, uniqueness: { scope: :domain }, format: { with: /\A#{SHORTCODE_RE_FRAGMENT}\z/ }, length: { minimum: 2 }
 
   scope :local,      -> { where(domain: nil) }
-- 
cgit 


From 898dacf83eaaec8ba02d4e44dd9901521d79a898 Mon Sep 17 00:00:00 2001
From: Eugen Rochko <eugen@zeonfederated.com>
Date: Thu, 8 Aug 2019 23:03:28 +0200
Subject: Fix weekly usage not being displayed correctly in hashtag admin UI
 (#11524)

Fix percentages in usage breakdown having too many digits

Change trending hashtags to only ask for review if a hashtag enters
the top 3 position in the set, since it's the only items shown in
the default web UI
---
 app/controllers/admin/tags_controller.rb | 2 +-
 app/models/trending_tags.rb              | 3 ++-
 app/views/admin/tags/show.html.haml      | 2 +-
 3 files changed, 4 insertions(+), 3 deletions(-)

(limited to 'app/models')

diff --git a/app/controllers/admin/tags_controller.rb b/app/controllers/admin/tags_controller.rb
index d62361eaa..39aca2a4b 100644
--- a/app/controllers/admin/tags_controller.rb
+++ b/app/controllers/admin/tags_controller.rb
@@ -71,7 +71,7 @@ module Admin
       now = Time.now.utc.beginning_of_day.to_date
 
       (Date.commercial(now.cwyear, now.cweek)..now).map do |date|
-        date.to_time.utc.beginning_of_day.to_i
+        date.to_time(:utc).beginning_of_day.to_i
       end
     end
   end
diff --git a/app/models/trending_tags.rb b/app/models/trending_tags.rb
index 594ae9520..3d60a7fea 100644
--- a/app/models/trending_tags.rb
+++ b/app/models/trending_tags.rb
@@ -6,6 +6,7 @@ class TrendingTags
   EXPIRE_TRENDS_AFTER  = 1.day.seconds
   THRESHOLD            = 5
   LIMIT                = 10
+  REVIEW_THRESHOLD     = 3
 
   class << self
     include Redisable
@@ -60,7 +61,7 @@ class TrendingTags
         old_rank = redis.zrevrank(key, tag.id)
 
         redis.zadd(key, score, tag.id)
-        request_review!(tag) if (old_rank.nil? || old_rank > LIMIT) && redis.zrevrank(key, tag.id) <= LIMIT && !tag.trendable? && tag.requires_review? && !tag.requested_review?
+        request_review!(tag) if (old_rank.nil? || old_rank > REVIEW_THRESHOLD) && redis.zrevrank(key, tag.id) <= REVIEW_THRESHOLD && !tag.trendable? && tag.requires_review? && !tag.requested_review?
       end
 
       redis.expire(key, EXPIRE_TRENDS_AFTER)
diff --git a/app/views/admin/tags/show.html.haml b/app/views/admin/tags/show.html.haml
index 6a1e03065..c3779d48c 100644
--- a/app/views/admin/tags/show.html.haml
+++ b/app/views/admin/tags/show.html.haml
@@ -41,5 +41,5 @@
       - @usage_by_domain.each do |(domain, count)|
         %tr
           %th= domain || site_hostname
-          %td= "#{number_with_delimiter((count.to_f / @tag.history[0][:uses].to_f) * 100)}%"
+          %td= number_to_percentage((count / @tag.history[0][:uses].to_f) * 100)
           %td= number_with_delimiter count
-- 
cgit 


From 7a1f8a58df7edeb4f1d03c9dd3c25d5370d858a6 Mon Sep 17 00:00:00 2001
From: Eugen Rochko <eugen@zeonfederated.com>
Date: Thu, 8 Aug 2019 23:04:19 +0200
Subject: Fix crash when saving invalid domain name (#11528)

Fix #7629
---
 app/models/account_domain_block.rb         |  2 +-
 app/models/concerns/domain_normalizable.rb |  2 +-
 app/models/domain_allow.rb                 |  2 +-
 app/models/domain_block.rb                 |  2 +-
 app/models/email_domain_block.rb           |  2 +-
 app/validators/domain_validator.rb         | 17 +++++++++++++++++
 config/locales/en.yml                      |  2 ++
 7 files changed, 24 insertions(+), 5 deletions(-)
 create mode 100644 app/validators/domain_validator.rb

(limited to 'app/models')

diff --git a/app/models/account_domain_block.rb b/app/models/account_domain_block.rb
index 7c0d60379..3aaffde9a 100644
--- a/app/models/account_domain_block.rb
+++ b/app/models/account_domain_block.rb
@@ -15,7 +15,7 @@ class AccountDomainBlock < ApplicationRecord
   include DomainNormalizable
 
   belongs_to :account
-  validates :domain, presence: true, uniqueness: { scope: :account_id }
+  validates :domain, presence: true, uniqueness: { scope: :account_id }, domain: true
 
   after_commit :remove_blocking_cache
   after_commit :remove_relationship_cache
diff --git a/app/models/concerns/domain_normalizable.rb b/app/models/concerns/domain_normalizable.rb
index fb84058fc..c00b3142f 100644
--- a/app/models/concerns/domain_normalizable.rb
+++ b/app/models/concerns/domain_normalizable.rb
@@ -4,7 +4,7 @@ module DomainNormalizable
   extend ActiveSupport::Concern
 
   included do
-    before_validation :normalize_domain
+    before_save :normalize_domain
   end
 
   private
diff --git a/app/models/domain_allow.rb b/app/models/domain_allow.rb
index 85018b636..5fe0e3a29 100644
--- a/app/models/domain_allow.rb
+++ b/app/models/domain_allow.rb
@@ -13,7 +13,7 @@
 class DomainAllow < ApplicationRecord
   include DomainNormalizable
 
-  validates :domain, presence: true, uniqueness: true
+  validates :domain, presence: true, uniqueness: true, domain: true
 
   scope :matches_domain, ->(value) { where(arel_table[:domain].matches("%#{value}%")) }
 
diff --git a/app/models/domain_block.rb b/app/models/domain_block.rb
index 3f5b9f23e..37b8d98c6 100644
--- a/app/models/domain_block.rb
+++ b/app/models/domain_block.rb
@@ -19,7 +19,7 @@ class DomainBlock < ApplicationRecord
 
   enum severity: [:silence, :suspend, :noop]
 
-  validates :domain, presence: true, uniqueness: true
+  validates :domain, presence: true, uniqueness: true, domain: true
 
   has_many :accounts, foreign_key: :domain, primary_key: :domain
   delegate :count, to: :accounts, prefix: true
diff --git a/app/models/email_domain_block.rb b/app/models/email_domain_block.rb
index 0fcd36477..bc70dea25 100644
--- a/app/models/email_domain_block.rb
+++ b/app/models/email_domain_block.rb
@@ -12,7 +12,7 @@
 class EmailDomainBlock < ApplicationRecord
   include DomainNormalizable
 
-  validates :domain, presence: true, uniqueness: true
+  validates :domain, presence: true, uniqueness: true, domain: true
 
   def self.block?(email)
     _, domain = email.split('@', 2)
diff --git a/app/validators/domain_validator.rb b/app/validators/domain_validator.rb
new file mode 100644
index 000000000..ae07f1798
--- /dev/null
+++ b/app/validators/domain_validator.rb
@@ -0,0 +1,17 @@
+# frozen_string_literal: true
+
+class DomainValidator < ActiveModel::EachValidator
+  def validate_each(record, attribute, value)
+    return if value.blank?
+
+    record.errors.add(attribute, I18n.t('domain_validator.invalid_domain')) unless compliant?(value)
+  end
+
+  private
+
+  def compliant?(value)
+    Addressable::URI.new.tap { |uri| uri.host = value }
+  rescue Addressable::URI::InvalidURIError
+    false
+  end
+end
diff --git a/config/locales/en.yml b/config/locales/en.yml
index 7fd0536ae..786906e2d 100644
--- a/config/locales/en.yml
+++ b/config/locales/en.yml
@@ -628,6 +628,8 @@ en:
     people:
       one: "%{count} person"
       other: "%{count} people"
+  domain_validator:
+    invalid_domain: is not a valid domain name
   errors:
     '403': You don't have permission to view this page.
     '404': The page you are looking for isn't here.
-- 
cgit 


From aebefc7ce9cff913c4f04bb94ebda428e755ecad Mon Sep 17 00:00:00 2001
From: kedama <kedama@foresdon.jp>
Date: Sun, 11 Aug 2019 20:15:18 +0900
Subject: Fix some emojis in profile metadata labels are not emojified.
 (#11534)

---
 app/models/account.rb | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

(limited to 'app/models')

diff --git a/app/models/account.rb b/app/models/account.rb
index b205c8c9e..60c06aaf0 100644
--- a/app/models/account.rb
+++ b/app/models/account.rb
@@ -506,7 +506,7 @@ class Account < ApplicationRecord
   end
 
   def emojifiable_text
-    [note, display_name, fields.map(&:value)].join(' ')
+    [note, display_name, fields.map(&:name), fields.map(&:value)].join(' ')
   end
 
   def clean_feed_manager
-- 
cgit 


From 597ea5687a3601c6cc462831bf570bae05b08240 Mon Sep 17 00:00:00 2001
From: Thibaut Girka <thib@sitedethib.com>
Date: Mon, 12 Aug 2019 15:31:20 +0200
Subject: Add option to disable emoji replacements

Fixes #647

The option is found in `/settings` (because that was easier to write it this
way) but only affects the glitch-soc front-end.
---
 app/controllers/settings/preferences_controller.rb       | 1 +
 app/javascript/flavours/glitch/util/emoji/index.js       | 6 +++---
 app/javascript/flavours/glitch/util/initial_state.js     | 1 +
 app/lib/user_settings_decorator.rb                       | 5 +++++
 app/models/user.rb                                       | 2 +-
 app/serializers/initial_state_serializer.rb              | 1 +
 app/views/settings/preferences/appearance/show.html.haml | 1 +
 config/locales/simple_form.en.yml                        | 1 +
 config/settings.yml                                      | 1 +
 9 files changed, 15 insertions(+), 4 deletions(-)

(limited to 'app/models')

diff --git a/app/controllers/settings/preferences_controller.rb b/app/controllers/settings/preferences_controller.rb
index 418ea5d7a..f05dbe0ea 100644
--- a/app/controllers/settings/preferences_controller.rb
+++ b/app/controllers/settings/preferences_controller.rb
@@ -49,6 +49,7 @@ class Settings::PreferencesController < Settings::BaseController
       :setting_expand_spoilers,
       :setting_reduce_motion,
       :setting_system_font_ui,
+      :setting_system_emoji_font,
       :setting_noindex,
       :setting_hide_network,
       :setting_hide_followers_count,
diff --git a/app/javascript/flavours/glitch/util/emoji/index.js b/app/javascript/flavours/glitch/util/emoji/index.js
index 2f40f9b08..b2d13cc95 100644
--- a/app/javascript/flavours/glitch/util/emoji/index.js
+++ b/app/javascript/flavours/glitch/util/emoji/index.js
@@ -1,4 +1,4 @@
-import { autoPlayGif } from 'flavours/glitch/util/initial_state';
+import { autoPlayGif, useSystemEmojiFont } from 'flavours/glitch/util/initial_state';
 import unicodeMapping from './emoji_unicode_mapping_light';
 import Trie from 'substring-trie';
 
@@ -12,7 +12,7 @@ const emojify = (str, customEmojis = {}) => {
   let rtn = '', tagChars = tagCharsWithEmojis, invisible = 0;
   for (;;) {
     let match, i = 0, tag;
-    while (i < str.length && (tag = tagChars.indexOf(str[i])) === -1 && (invisible || !(match = trie.search(str.slice(i))))) {
+    while (i < str.length && (tag = tagChars.indexOf(str[i])) === -1 && (invisible || useSystemEmojiFont || !(match = trie.search(str.slice(i))))) {
       i += str.codePointAt(i) < 65536 ? 1 : 2;
     }
     let rend, replacement = '';
@@ -57,7 +57,7 @@ const emojify = (str, customEmojis = {}) => {
         }
       }
       i = rend;
-    } else { // matched to unicode emoji
+    } else if (!useSystemEmojiFont) { // matched to unicode emoji
       const { filename, shortCode } = unicodeMapping[match];
       const title = shortCode ? `:${shortCode}:` : '';
       replacement = `<img draggable="false" class="emojione" alt="${match}" title="${title}" src="${assetHost}/emoji/${filename}.svg" />`;
diff --git a/app/javascript/flavours/glitch/util/initial_state.js b/app/javascript/flavours/glitch/util/initial_state.js
index caaa79bb3..4b6227cac 100644
--- a/app/javascript/flavours/glitch/util/initial_state.js
+++ b/app/javascript/flavours/glitch/util/initial_state.js
@@ -31,5 +31,6 @@ export const defaultContentType = getMeta('default_content_type');
 export const forceSingleColumn = getMeta('advanced_layout') === false;
 export const useBlurhash = getMeta('use_blurhash');
 export const usePendingItems = getMeta('use_pending_items');
+export const useSystemEmojiFont = getMeta('system_emoji_font');
 
 export default initialState;
diff --git a/app/lib/user_settings_decorator.rb b/app/lib/user_settings_decorator.rb
index a52172707..c822d54de 100644
--- a/app/lib/user_settings_decorator.rb
+++ b/app/lib/user_settings_decorator.rb
@@ -29,6 +29,7 @@ class UserSettingsDecorator
     user.settings['expand_spoilers']     = expand_spoilers_preference if change?('setting_expand_spoilers')
     user.settings['reduce_motion']       = reduce_motion_preference if change?('setting_reduce_motion')
     user.settings['system_font_ui']      = system_font_ui_preference if change?('setting_system_font_ui')
+    user.settings['system_emoji_font']   = system_emoji_font_preference if change?('setting_system_emoji_font')
     user.settings['noindex']             = noindex_preference if change?('setting_noindex')
     user.settings['hide_followers_count']= hide_followers_count_preference if change?('setting_hide_followers_count')
     user.settings['flavour']             = flavour_preference if change?('setting_flavour')
@@ -79,6 +80,10 @@ class UserSettingsDecorator
     boolean_cast_setting 'setting_system_font_ui'
   end
 
+  def system_emoji_font_preference
+    boolean_cast_setting 'setting_system_emoji_font'
+  end
+
   def auto_play_gif_preference
     boolean_cast_setting 'setting_auto_play_gif'
   end
diff --git a/app/models/user.rb b/app/models/user.rb
index 45a4b8989..341227d3e 100644
--- a/app/models/user.rb
+++ b/app/models/user.rb
@@ -108,7 +108,7 @@ class User < ApplicationRecord
            :reduce_motion, :system_font_ui, :noindex, :flavour, :skin, :display_media, :hide_network, :hide_followers_count,
            :expand_spoilers, :default_language, :aggregate_reblogs, :show_application,
            :advanced_layout, :use_blurhash, :use_pending_items, :trends,
-           :default_content_type,
+           :default_content_type, :system_emoji_font,
            to: :settings, prefix: :setting, allow_nil: false
 
   attr_reader :invite_code
diff --git a/app/serializers/initial_state_serializer.rb b/app/serializers/initial_state_serializer.rb
index c8da6e725..c8f6bec7a 100644
--- a/app/serializers/initial_state_serializer.rb
+++ b/app/serializers/initial_state_serializer.rb
@@ -53,6 +53,7 @@ class InitialStateSerializer < ActiveModel::Serializer
       store[:is_staff]          = object.current_account.user.staff?
       store[:trends]            = Setting.trends && object.current_account.user.setting_trends
       store[:default_content_type] = object.current_account.user.setting_default_content_type
+      store[:system_emoji_font] = object.current_account.user.setting_system_emoji_font
     end
 
     store
diff --git a/app/views/settings/preferences/appearance/show.html.haml b/app/views/settings/preferences/appearance/show.html.haml
index 0bda49f44..900a7c6fb 100644
--- a/app/views/settings/preferences/appearance/show.html.haml
+++ b/app/views/settings/preferences/appearance/show.html.haml
@@ -21,6 +21,7 @@
     = f.input :setting_auto_play_gif, as: :boolean, wrapper: :with_label, recommended: true
     = f.input :setting_reduce_motion, as: :boolean, wrapper: :with_label
     = f.input :setting_system_font_ui, as: :boolean, wrapper: :with_label
+    = f.input :setting_system_emoji_font, as: :boolean, wrapper: :with_label
 
   %h4= t 'appearance.discovery'
 
diff --git a/config/locales/simple_form.en.yml b/config/locales/simple_form.en.yml
index 5da0cc45d..8f63df013 100644
--- a/config/locales/simple_form.en.yml
+++ b/config/locales/simple_form.en.yml
@@ -124,6 +124,7 @@ en:
         setting_reduce_motion: Reduce motion in animations
         setting_show_application: Disclose application used to send toots
         setting_skin: Skin
+        setting_system_emoji_font: Use system's default font for emojis (applies to Glitch flavour only)
         setting_system_font_ui: Use system's default font
         setting_theme: Site theme
         setting_trends: Show today's trends
diff --git a/config/settings.yml b/config/settings.yml
index 10836db3f..70a9a3a44 100644
--- a/config/settings.yml
+++ b/config/settings.yml
@@ -29,6 +29,7 @@ defaults: &defaults
   reduce_motion: false
   show_application: false
   system_font_ui: false
+  system_emoji_font: false
   noindex: false
   hide_followers_count: false
   enable_keybase: true
-- 
cgit 


From d7bdddbeef3df2fa61d2eee42b544b6553d2d2ea Mon Sep 17 00:00:00 2001
From: Arnout Engelen <github@bzzt.net>
Date: Thu, 15 Aug 2019 20:20:20 +0200
Subject: Include max image dimensions in error (#11552)

---
 app/models/concerns/attachmentable.rb | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

(limited to 'app/models')

diff --git a/app/models/concerns/attachmentable.rb b/app/models/concerns/attachmentable.rb
index 7c78bb456..246c2c27c 100644
--- a/app/models/concerns/attachmentable.rb
+++ b/app/models/concerns/attachmentable.rb
@@ -43,7 +43,7 @@ module Attachmentable
 
       width, height = FastImage.size(attachment.queued_for_write[:original].path)
 
-      raise Mastodon::DimensionsValidationError, "#{width}x#{height} images are not supported" if width.present? && height.present? && (width * height >= MAX_MATRIX_LIMIT)
+      raise Mastodon::DimensionsValidationError, "#{width}x#{height} images are not supported, must be below #{MAX_MATRIX_LIMIT} sqpx" if width.present? && height.present? && (width * height >= MAX_MATRIX_LIMIT)
     end
   end
 
-- 
cgit 


From 8fdff2748ffbc4ce3d36c75f2bd33182b641e895 Mon Sep 17 00:00:00 2001
From: Eugen Rochko <eugen@zeonfederated.com>
Date: Fri, 16 Aug 2019 01:24:03 +0200
Subject: Add more accurate account search (#11537)

* Add more accurate account search

When ElasticSearch is available, a more accurate search is implemented:

- Using edge n-gram index for acct and display name
- Using asciifolding and cjk width normalization on display names
- Using Gaussian decay on account activity for additional scoring (recency)
- Using followers/friends ratio for additional scoring (spamminess)
- Using followers number for additional scoring (size)

The exact match precedence only takes effect when the input conforms
to the username format and the username part of it is complete, i.e.
when the user started typing the domain part.

* Support single-letter usernames

* Fix tests

* Fix not picking up account updates

* Add weights and normalization for scores, skip zero terms queries

* Use local counts for accounts index, adjust search parameters

* Fix mistakes

* Using updated_at of accounts is inadequate for remote accounts
---
 app/chewy/accounts_index.rb                  |  36 +++++
 app/models/account.rb                        |   6 +
 app/models/account_stat.rb                   |   2 +
 app/services/account_search_service.rb       | 189 ++++++++++++++++++---------
 spec/services/account_search_service_spec.rb | 122 ++++-------------
 5 files changed, 194 insertions(+), 161 deletions(-)
 create mode 100644 app/chewy/accounts_index.rb

(limited to 'app/models')

diff --git a/app/chewy/accounts_index.rb b/app/chewy/accounts_index.rb
new file mode 100644
index 000000000..e11b80039
--- /dev/null
+++ b/app/chewy/accounts_index.rb
@@ -0,0 +1,36 @@
+# frozen_string_literal: true
+
+class AccountsIndex < Chewy::Index
+  settings index: { refresh_interval: '5m' }, analysis: {
+    analyzer: {
+      content: {
+        tokenizer: 'whitespace',
+        filter: %w(lowercase asciifolding cjk_width),
+      },
+
+      edge_ngram: {
+        tokenizer: 'edge_ngram',
+        filter: %w(lowercase asciifolding cjk_width),
+      },
+    },
+
+    tokenizer: {
+      edge_ngram: {
+        type: 'edge_ngram',
+        min_gram: 1,
+        max_gram: 15,
+      },
+    },
+  }
+
+  define_type ::Account.searchable.includes(:account_stat), delete_if: ->(account) { account.destroyed? || !account.searchable? } do
+    root date_detection: false do
+      field :id, type: 'long'
+      field :display_name, type: 'text', analyzer: 'edge_ngram', search_analyzer: 'content'
+      field :acct, type: 'text', analyzer: 'edge_ngram', search_analyzer: 'content', value: ->(account) { [account.username, account.domain].compact.join('@') }
+      field :following_count, type: 'long', value: ->(account) { account.active_relationships.count }
+      field :followers_count, type: 'long', value: ->(account) { account.passive_relationships.count }
+      field :last_status_at, type: 'date', value: ->(account) { account.last_status_at || account.created_at }
+    end
+  end
+end
diff --git a/app/models/account.rb b/app/models/account.rb
index 60c06aaf0..392cc625f 100644
--- a/app/models/account.rb
+++ b/app/models/account.rb
@@ -127,6 +127,8 @@ class Account < ApplicationRecord
 
   delegate :chosen_languages, to: :user, prefix: false, allow_nil: true
 
+  update_index('accounts#account', :self) if Chewy.enabled?
+
   def local?
     domain.nil?
   end
@@ -169,6 +171,10 @@ class Account < ApplicationRecord
     subscription_expires_at.present?
   end
 
+  def searchable?
+    !(suspended? || moved?)
+  end
+
   def possibly_stale?
     last_webfingered_at.nil? || last_webfingered_at <= 1.day.ago
   end
diff --git a/app/models/account_stat.rb b/app/models/account_stat.rb
index 9813aa84f..6d1097cec 100644
--- a/app/models/account_stat.rb
+++ b/app/models/account_stat.rb
@@ -16,6 +16,8 @@
 class AccountStat < ApplicationRecord
   belongs_to :account, inverse_of: :account_stat
 
+  update_index('accounts#account', :account) if Chewy.enabled?
+
   def increment_count!(key)
     update(attributes_for_increment(key))
   end
diff --git a/app/services/account_search_service.rb b/app/services/account_search_service.rb
index e1874d045..2d602a31d 100644
--- a/app/services/account_search_service.rb
+++ b/app/services/account_search_service.rb
@@ -4,105 +4,134 @@ class AccountSearchService < BaseService
   attr_reader :query, :limit, :offset, :options, :account
 
   def call(query, account = nil, options = {})
-    @query   = query.strip
-    @limit   = options[:limit].to_i
-    @offset  = options[:offset].to_i
-    @options = options
-    @account = account
+    @acct_hint = query.start_with?('@')
+    @query     = query.strip.gsub(/\A@/, '')
+    @limit     = options[:limit].to_i
+    @offset    = options[:offset].to_i
+    @options   = options
+    @account   = account
 
-    search_service_results
+    search_service_results.compact.uniq
   end
 
   private
 
   def search_service_results
-    return [] if query_blank_or_hashtag? || limit < 1
+    return [] if query.blank? || limit < 1
 
-    if resolving_non_matching_remote_account?
-      [ResolveAccountService.new.call("#{query_username}@#{query_domain}")].compact
-    else
-      search_results_and_exact_match.compact.uniq
-    end
+    [exact_match] + search_results
   end
 
-  def resolving_non_matching_remote_account?
-    offset.zero? && options[:resolve] && !exact_match? && !domain_is_local?
-  end
+  def exact_match
+    return unless offset.zero? && username_complete?
 
-  def search_results_and_exact_match
-    return search_results.to_a unless offset.zero?
+    return @exact_match if defined?(@exact_match)
 
-    results = [exact_match]
+    @exact_match = begin
+      if options[:resolve]
+        ResolveAccountService.new.call(query)
+      elsif domain_is_local?
+        Account.find_local(query_username)
+      else
+        Account.find_remote(query_username, query_domain)
+      end
+    end
+  end
 
-    return results if exact_match? && limit == 1
+  def search_results
+    return [] if limit_for_non_exact_results.zero?
 
-    results + search_results.to_a
+    @search_results ||= begin
+      if Chewy.enabled?
+        from_elasticsearch
+      else
+        from_database
+      end
+    end
   end
 
-  def query_blank_or_hashtag?
-    query.blank? || query.start_with?('#')
+  def from_database
+    if account
+      advanced_search_results
+    else
+      simple_search_results
+    end
   end
 
-  def split_query_string
-    @split_query_string ||= query.gsub(/\A@/, '').split('@')
+  def advanced_search_results
+    Account.advanced_search_for(terms_for_query, account, limit_for_non_exact_results, options[:following], offset)
   end
 
-  def query_username
-    @query_username ||= split_query_string.first || ''
+  def simple_search_results
+    Account.search_for(terms_for_query, limit_for_non_exact_results, offset)
   end
 
-  def query_domain
-    @query_domain ||= query_without_split? ? nil : split_query_string.last
-  end
+  def from_elasticsearch
+    must_clauses   = [{ multi_match: { query: terms_for_query, fields: likely_acct? ? %w(acct) : %w(acct^2 display_name), type: 'best_fields' } }]
+    should_clauses = []
 
-  def query_without_split?
-    split_query_string.size == 1
-  end
+    if account
+      return [] if options[:following] && following_ids.empty?
 
-  def domain_is_local?
-    @domain_is_local ||= TagManager.instance.local_domain?(query_domain)
-  end
+      if options[:following]
+        must_clauses << { terms: { id: following_ids } }
+      elsif following_ids.any?
+        should_clauses << { terms: { id: following_ids, boost: 100 } }
+      end
+    end
 
-  def search_from
-    options[:following] && account ? account.following : Account
-  end
+    query     = { bool: { must: must_clauses, should: should_clauses } }
+    functions = [reputation_score_function, followers_score_function, time_distance_function]
 
-  def exact_match?
-    exact_match.present?
-  end
+    records = AccountsIndex.query(function_score: { query: query, functions: functions, boost_mode: 'multiply', score_mode: 'avg' })
+                           .limit(limit_for_non_exact_results)
+                           .offset(offset)
+                           .objects
+                           .compact
 
-  def exact_match
-    return @exact_match if defined?(@exact_match)
+    ActiveRecord::Associations::Preloader.new.preload(records, :account_stat)
 
-    @exact_match = begin
-      if domain_is_local?
-        search_from.without_suspended.find_local(query_username)
-      else
-        search_from.without_suspended.find_remote(query_username, query_domain)
-      end
-    end
+    records
   end
 
-  def search_results
-    @search_results ||= begin
-      if account
-        advanced_search_results
-      else
-        simple_search_results
-      end
-    end
+  def reputation_score_function
+    {
+      script_score: {
+        script: {
+          source: "(doc['followers_count'].value + 0.0) / (doc['followers_count'].value + doc['following_count'].value + 1)",
+        },
+      },
+    }
   end
 
-  def advanced_search_results
-    Account.advanced_search_for(terms_for_query, account, limit_for_non_exact_results, options[:following], offset)
+  def followers_score_function
+    {
+      field_value_factor: {
+        field: 'followers_count',
+        modifier: 'log2p',
+        missing: 1,
+      },
+    }
   end
 
-  def simple_search_results
-    Account.search_for(terms_for_query, limit_for_non_exact_results, offset)
+  def time_distance_function
+    {
+      gauss: {
+        last_status_at: {
+          scale: '30d',
+          offset: '30d',
+          decay: 0.3,
+        },
+      },
+    }
+  end
+
+  def following_ids
+    @following_ids ||= account.active_relationships.pluck(:target_account_id)
   end
 
   def limit_for_non_exact_results
-    if offset.zero? && exact_match?
+    if exact_match?
       limit - 1
     else
       limit
@@ -113,7 +142,39 @@ class AccountSearchService < BaseService
     if domain_is_local?
       query_username
     else
-      "#{query_username} #{query_domain}"
+      query
     end
   end
+
+  def split_query_string
+    @split_query_string ||= query.split('@')
+  end
+
+  def query_username
+    @query_username ||= split_query_string.first || ''
+  end
+
+  def query_domain
+    @query_domain ||= query_without_split? ? nil : split_query_string.last
+  end
+
+  def query_without_split?
+    split_query_string.size == 1
+  end
+
+  def domain_is_local?
+    @domain_is_local ||= TagManager.instance.local_domain?(query_domain)
+  end
+
+  def exact_match?
+    exact_match.present?
+  end
+
+  def username_complete?
+    query.include?('@') && "@#{query}" =~ Account::MENTION_RE
+  end
+
+  def likely_acct?
+    @acct_hint || username_complete?
+  end
 end
diff --git a/spec/services/account_search_service_spec.rb b/spec/services/account_search_service_spec.rb
index 7b071b378..5b7182586 100644
--- a/spec/services/account_search_service_spec.rb
+++ b/spec/services/account_search_service_spec.rb
@@ -1,126 +1,56 @@
 require 'rails_helper'
 
 describe AccountSearchService, type: :service do
-  describe '.call' do
-    describe 'with a query to ignore' do
+  describe '#call' do
+    context 'with a query to ignore' do
       it 'returns empty array for missing query' do
         results = subject.call('', nil, limit: 10)
 
         expect(results).to eq []
       end
-      it 'returns empty array for hashtag query' do
-        results = subject.call('#tag', nil, limit: 10)
 
-        expect(results).to eq []
-      end
       it 'returns empty array for limit zero' do
         Fabricate(:account, username: 'match')
+
         results = subject.call('match', nil, limit: 0)
 
         expect(results).to eq []
       end
     end
 
-    describe 'searching for a simple term that is not an exact match' do
+    context 'searching for a simple term that is not an exact match' do
       it 'does not return a nil entry in the array for the exact match' do
-        match = Fabricate(:account, username: 'matchingusername')
-
+        account = Fabricate(:account, username: 'matchingusername')
         results = subject.call('match', nil, limit: 5)
-        expect(results).to eq [match]
-      end
-    end
 
-    describe 'searching local and remote users' do
-      describe "when only '@'" do
-        before do
-          allow(Account).to receive(:find_local)
-          allow(Account).to receive(:search_for)
-          subject.call('@', nil, limit: 10)
-        end
-
-        it 'uses find_local with empty query to look for local accounts' do
-          expect(Account).to have_received(:find_local).with('')
-        end
-      end
-
-      describe 'when no domain' do
-        before do
-          allow(Account).to receive(:find_local)
-          allow(Account).to receive(:search_for)
-          subject.call('one', nil, limit: 10)
-        end
-
-        it 'uses find_local to look for local accounts' do
-          expect(Account).to have_received(:find_local).with('one')
-        end
-
-        it 'uses search_for to find matches' do
-          expect(Account).to have_received(:search_for).with('one', 10, 0)
-        end
-      end
-
-      describe 'when there is a domain' do
-        before do
-          allow(Account).to receive(:find_remote)
-        end
-
-        it 'uses find_remote to look for remote accounts' do
-          subject.call('two@example.com', nil, limit: 10)
-          expect(Account).to have_received(:find_remote).with('two', 'example.com')
-        end
-
-        describe 'and there is no account provided' do
-          it 'uses search_for to find matches' do
-            allow(Account).to receive(:search_for)
-            subject.call('two@example.com', nil, limit: 10, resolve: false)
-
-            expect(Account).to have_received(:search_for).with('two example.com', 10, 0)
-          end
-        end
-
-        describe 'and there is an account provided' do
-          it 'uses advanced_search_for to find matches' do
-            account = Fabricate(:account)
-            allow(Account).to receive(:advanced_search_for)
-            subject.call('two@example.com', account, limit: 10, resolve: false)
-
-            expect(Account).to have_received(:advanced_search_for).with('two example.com', account, 10, nil, 0)
-          end
-        end
+        expect(results).to eq [account]
       end
     end
 
-    describe 'with an exact match' do
-      it 'returns exact match first, and does not return duplicates' do
-        partial = Fabricate(:account, username: 'exactness')
-        exact = Fabricate(:account, username: 'exact')
-
-        results = subject.call('exact', nil, limit: 10)
-        expect(results.size).to eq 2
-        expect(results).to eq [exact, partial]
-      end
-    end
-
-    describe 'when there is a local domain' do
+    context 'when there is a local domain' do
       around do |example|
         before = Rails.configuration.x.local_domain
+
         example.run
+
         Rails.configuration.x.local_domain = before
       end
 
       it 'returns exact match first' do
         remote     = Fabricate(:account, username: 'a', domain: 'remote', display_name: 'e')
         remote_too = Fabricate(:account, username: 'b', domain: 'remote', display_name: 'e')
-        exact = Fabricate(:account, username: 'e')
+        exact      = Fabricate(:account, username: 'e')
+
         Rails.configuration.x.local_domain = 'example.com'
 
         results = subject.call('e@example.com', nil, limit: 2)
+
         expect(results.size).to eq 2
         expect(results).to eq([exact, remote]).or eq([exact, remote_too])
       end
     end
 
-    describe 'when there is a domain but no exact match' do
+    context 'when there is a domain but no exact match' do
       it 'follows the remote account when resolve is true' do
         service = double(call: nil)
         allow(ResolveAccountService).to receive(:new).and_return(service)
@@ -138,23 +68,21 @@ describe AccountSearchService, type: :service do
       end
     end
 
-    describe 'should not include suspended accounts' do
-      it 'returns the fuzzy match first, and does not return suspended exacts' do
-        partial = Fabricate(:account, username: 'exactness')
-        exact = Fabricate(:account, username: 'exact', suspended: true)
+    it 'returns the fuzzy match first, and does not return suspended exacts' do
+      partial = Fabricate(:account, username: 'exactness')
+      exact   = Fabricate(:account, username: 'exact', suspended: true)
+      results = subject.call('exact', nil, limit: 10)
 
-        results = subject.call('exact', nil, limit: 10)
-        expect(results.size).to eq 1
-        expect(results).to eq [partial]
-      end
+      expect(results.size).to eq 1
+      expect(results).to eq [partial]
+    end
 
-      it "does not return suspended remote accounts" do
-        remote = Fabricate(:account, username: 'a', domain: 'remote', display_name: 'e', suspended: true)
+    it "does not return suspended remote accounts" do
+      remote  = Fabricate(:account, username: 'a', domain: 'remote', display_name: 'e', suspended: true)
+      results = subject.call('a@example.com', nil, limit: 2)
 
-        results = subject.call('a@example.com', nil, limit: 2)
-        expect(results.size).to eq 0
-        expect(results).to eq []
-      end
+      expect(results.size).to eq 0
+      expect(results).to eq []
     end
   end
 end
-- 
cgit 


From 5d8ee24cd5d82d41a6914ee9982c38cc18d859b4 Mon Sep 17 00:00:00 2001
From: Stanislas <angristan@pm.me>
Date: Sat, 17 Aug 2019 22:04:15 +0200
Subject: Remove WebP support (#11589)

---
 app/javascript/mastodon/utils/resize_image.js | 2 +-
 app/models/concerns/account_avatar.rb         | 2 +-
 app/models/concerns/account_header.rb         | 2 +-
 app/models/custom_emoji.rb                    | 2 +-
 app/models/media_attachment.rb                | 4 ++--
 app/models/preview_card.rb                    | 2 +-
 6 files changed, 7 insertions(+), 7 deletions(-)

(limited to 'app/models')

diff --git a/app/javascript/mastodon/utils/resize_image.js b/app/javascript/mastodon/utils/resize_image.js
index d566edb03..7196dc96b 100644
--- a/app/javascript/mastodon/utils/resize_image.js
+++ b/app/javascript/mastodon/utils/resize_image.js
@@ -31,7 +31,7 @@ const loadImage = inputFile => new Promise((resolve, reject) => {
 });
 
 const getOrientation = (img, type = 'image/png') => new Promise(resolve => {
-  if (!['image/jpeg', 'image/webp'].includes(type)) {
+  if (type !== 'image/jpeg') {
     resolve(1);
     return;
   }
diff --git a/app/models/concerns/account_avatar.rb b/app/models/concerns/account_avatar.rb
index 5fff3ef5d..2d5ebfca3 100644
--- a/app/models/concerns/account_avatar.rb
+++ b/app/models/concerns/account_avatar.rb
@@ -3,7 +3,7 @@
 module AccountAvatar
   extend ActiveSupport::Concern
 
-  IMAGE_MIME_TYPES = ['image/jpeg', 'image/png', 'image/gif', 'image/webp'].freeze
+  IMAGE_MIME_TYPES = ['image/jpeg', 'image/png', 'image/gif'].freeze
   LIMIT = 2.megabytes
 
   class_methods do
diff --git a/app/models/concerns/account_header.rb b/app/models/concerns/account_header.rb
index a748fdff7..067e166eb 100644
--- a/app/models/concerns/account_header.rb
+++ b/app/models/concerns/account_header.rb
@@ -3,7 +3,7 @@
 module AccountHeader
   extend ActiveSupport::Concern
 
-  IMAGE_MIME_TYPES = ['image/jpeg', 'image/png', 'image/gif', 'image/webp'].freeze
+  IMAGE_MIME_TYPES = ['image/jpeg', 'image/png', 'image/gif'].freeze
   LIMIT = 2.megabytes
   MAX_PIXELS = 750_000 # 1500x500px
 
diff --git a/app/models/custom_emoji.rb b/app/models/custom_emoji.rb
index 643a7e46a..b21ad9042 100644
--- a/app/models/custom_emoji.rb
+++ b/app/models/custom_emoji.rb
@@ -28,7 +28,7 @@ class CustomEmoji < ApplicationRecord
     :(#{SHORTCODE_RE_FRAGMENT}):
     (?=[^[:alnum:]:]|$)/x
 
-  IMAGE_MIME_TYPES = %w(image/png image/gif image/webp).freeze
+  IMAGE_MIME_TYPES = %w(image/png image/gif).freeze
 
   belongs_to :category, class_name: 'CustomEmojiCategory', optional: true
   has_one :local_counterpart, -> { where(domain: nil) }, class_name: 'CustomEmoji', primary_key: :shortcode, foreign_key: :shortcode
diff --git a/app/models/media_attachment.rb b/app/models/media_attachment.rb
index 2fc5ccf27..fe76b36aa 100644
--- a/app/models/media_attachment.rb
+++ b/app/models/media_attachment.rb
@@ -26,11 +26,11 @@ class MediaAttachment < ApplicationRecord
 
   enum type: [:image, :gifv, :video, :unknown, :audio]
 
-  IMAGE_FILE_EXTENSIONS = %w(.jpg .jpeg .png .gif .webp).freeze
+  IMAGE_FILE_EXTENSIONS = %w(.jpg .jpeg .png .gif).freeze
   VIDEO_FILE_EXTENSIONS = %w(.webm .mp4 .m4v .mov).freeze
   AUDIO_FILE_EXTENSIONS = %w(.ogg .oga .mp3 .wav .flac .opus .aac .m4a .3gp).freeze
 
-  IMAGE_MIME_TYPES             = %w(image/jpeg image/png image/gif image/webp).freeze
+  IMAGE_MIME_TYPES             = %w(image/jpeg image/png image/gif).freeze
   VIDEO_MIME_TYPES             = %w(video/webm video/mp4 video/quicktime video/ogg).freeze
   VIDEO_CONVERTIBLE_MIME_TYPES = %w(video/webm video/quicktime).freeze
   AUDIO_MIME_TYPES             = %w(audio/wave audio/wav audio/x-wav audio/x-pn-wave audio/ogg audio/mpeg audio/mp3 audio/webm audio/flac audio/aac audio/m4a audio/3gpp).freeze
diff --git a/app/models/preview_card.rb b/app/models/preview_card.rb
index f26ea0c74..a792b352b 100644
--- a/app/models/preview_card.rb
+++ b/app/models/preview_card.rb
@@ -25,7 +25,7 @@
 #
 
 class PreviewCard < ApplicationRecord
-  IMAGE_MIME_TYPES = ['image/jpeg', 'image/png', 'image/gif', 'image/webp'].freeze
+  IMAGE_MIME_TYPES = ['image/jpeg', 'image/png', 'image/gif'].freeze
   LIMIT = 1.megabytes
 
   self.inheritance_column = false
-- 
cgit 


From cc0a55cf9aead00e1cb044649f84c2187e0e4a35 Mon Sep 17 00:00:00 2001
From: Eugen Rochko <eugen@zeonfederated.com>
Date: Sun, 18 Aug 2019 03:45:51 +0200
Subject: Add more accurate hashtag search (#11579)

* Add more accurate hashtag search

Using ElasticSearch to index hashtags with edge n-grams and score
them by usage within the last 7 days since last activity. Only
hashtags that have been reviewed and are listable can appear in
searches, unless they match the query exactly

* Fix search analyzer dropping non-ascii characters
---
 app/chewy/tags_index.rb                            | 37 ++++++++++
 app/models/tag.rb                                  | 14 ++--
 app/models/trending_tags.rb                        |  3 +
 app/services/account_search_service.rb             |  2 +-
 app/services/search_service.rb                     |  8 +--
 app/services/tag_search_service.rb                 | 82 ++++++++++++++++++++++
 config/locales/simple_form.en.yml                  |  2 +-
 .../20190815225426_add_last_status_at_to_tags.rb   |  6 ++
 db/schema.rb                                       |  4 +-
 spec/models/tag_spec.rb                            |  4 +-
 10 files changed, 149 insertions(+), 13 deletions(-)
 create mode 100644 app/chewy/tags_index.rb
 create mode 100644 app/services/tag_search_service.rb
 create mode 100644 db/migrate/20190815225426_add_last_status_at_to_tags.rb

(limited to 'app/models')

diff --git a/app/chewy/tags_index.rb b/app/chewy/tags_index.rb
new file mode 100644
index 000000000..300fc128f
--- /dev/null
+++ b/app/chewy/tags_index.rb
@@ -0,0 +1,37 @@
+# frozen_string_literal: true
+
+class TagsIndex < Chewy::Index
+  settings index: { refresh_interval: '15m' }, analysis: {
+    analyzer: {
+      content: {
+        tokenizer: 'keyword',
+        filter: %w(lowercase asciifolding cjk_width),
+      },
+
+      edge_ngram: {
+        tokenizer: 'edge_ngram',
+        filter: %w(lowercase asciifolding cjk_width),
+      },
+    },
+
+    tokenizer: {
+      edge_ngram: {
+        type: 'edge_ngram',
+        min_gram: 2,
+        max_gram: 15,
+      },
+    },
+  }
+
+  define_type ::Tag.listable, delete_if: ->(tag) { tag.destroyed? || !tag.listable? } do
+    root date_detection: false do
+      field :name, type: 'text', analyzer: 'content' do
+        field :edge_ngram, type: 'text', analyzer: 'edge_ngram', search_analyzer: 'content'
+      end
+
+      field :reviewed, type: 'boolean', value: ->(tag) { tag.reviewed? }
+      field :usage, type: 'long', value: ->(tag) { tag.history.reduce(0) { |total, day| total + day[:accounts].to_i } }
+      field :last_status_at, type: 'date', value: ->(tag) { tag.last_status_at || tag.created_at }
+    end
+  end
+end
diff --git a/app/models/tag.rb b/app/models/tag.rb
index 1364d1dba..5094d973d 100644
--- a/app/models/tag.rb
+++ b/app/models/tag.rb
@@ -13,6 +13,8 @@
 #  listable            :boolean
 #  reviewed_at         :datetime
 #  requested_review_at :datetime
+#  last_status_at      :datetime
+#  last_trend_at       :datetime
 #
 
 class Tag < ApplicationRecord
@@ -33,7 +35,8 @@ class Tag < ApplicationRecord
   scope :unreviewed, -> { where(reviewed_at: nil) }
   scope :pending_review, -> { unreviewed.where.not(requested_review_at: nil) }
   scope :usable, -> { where(usable: [true, nil]) }
-  scope :discoverable, -> { where(listable: [true, nil]).joins(:account_tag_stat).where(AccountTagStat.arel_table[:accounts_count].gt(0)).order(Arel.sql('account_tag_stats.accounts_count desc')) }
+  scope :listable, -> { where(listable: [true, nil]) }
+  scope :discoverable, -> { listable.joins(:account_tag_stat).where(AccountTagStat.arel_table[:accounts_count].gt(0)).order(Arel.sql('account_tag_stats.accounts_count desc')) }
   scope :most_used, ->(account) { joins(:statuses).where(statuses: { account: account }).group(:id).order(Arel.sql('count(*) desc')) }
 
   delegate :accounts_count,
@@ -44,6 +47,8 @@ class Tag < ApplicationRecord
 
   after_save :save_account_tag_stat
 
+  update_index('tags#tag', :self) if Chewy.enabled?
+
   def account_tag_stat
     super || build_account_tag_stat
   end
@@ -121,9 +126,10 @@ class Tag < ApplicationRecord
       normalized_term = normalize(term.strip).mb_chars.downcase.to_s
       pattern         = sanitize_sql_like(normalized_term) + '%'
 
-      Tag.where(arel_table[:name].lower.matches(pattern))
-         .where(arel_table[:score].gt(0).or(arel_table[:name].lower.eq(normalized_term)))
-         .order(Arel.sql('length(name) ASC, score DESC, name ASC'))
+      Tag.listable
+         .where(arel_table[:name].lower.matches(pattern))
+         .where(arel_table[:name].lower.eq(normalized_term).or(arel_table[:reviewed_at].not_eq(nil)))
+         .order(Arel.sql('length(name) ASC, name ASC'))
          .limit(limit)
          .offset(offset)
     end
diff --git a/app/models/trending_tags.rb b/app/models/trending_tags.rb
index 3d60a7fea..e4ce988c1 100644
--- a/app/models/trending_tags.rb
+++ b/app/models/trending_tags.rb
@@ -17,6 +17,9 @@ class TrendingTags
       increment_historical_use!(tag.id, at_time)
       increment_unique_use!(tag.id, account.id, at_time)
       increment_vote!(tag, at_time)
+
+      tag.update(last_status_at: Time.now.utc) if tag.last_status_at.nil? || tag.last_status_at < 12.hours.ago
+      tag.update(last_trend_at: Time.now.utc)  if trending?(tag) && (tag.last_trend_at.nil? || tag.last_trend_at < 12.hours.ago)
     end
 
     def get(limit, filtered: true)
diff --git a/app/services/account_search_service.rb b/app/services/account_search_service.rb
index d7bccdfe0..01caaefa9 100644
--- a/app/services/account_search_service.rb
+++ b/app/services/account_search_service.rb
@@ -109,7 +109,7 @@ class AccountSearchService < BaseService
       field_value_factor: {
         field: 'followers_count',
         modifier: 'log2p',
-        missing: 1,
+        missing: 0,
       },
     }
   end
diff --git a/app/services/search_service.rb b/app/services/search_service.rb
index 786d34b15..fe601bbf4 100644
--- a/app/services/search_service.rb
+++ b/app/services/search_service.rb
@@ -57,10 +57,10 @@ class SearchService < BaseService
   end
 
   def perform_hashtags_search!
-    Tag.search_for(
-      @query.gsub(/\A#/, ''),
-      @limit,
-      @offset
+    TagSearchService.new.call(
+      @query,
+      limit: @limit,
+      offset: @offset
     )
   end
 
diff --git a/app/services/tag_search_service.rb b/app/services/tag_search_service.rb
new file mode 100644
index 000000000..64dd76bb7
--- /dev/null
+++ b/app/services/tag_search_service.rb
@@ -0,0 +1,82 @@
+# frozen_string_literal: true
+
+class TagSearchService < BaseService
+  def call(query, options = {})
+    @query  = query.strip.gsub(/\A#/, '')
+    @offset = options[:offset].to_i
+    @limit  = options[:limit].to_i
+
+    if Chewy.enabled?
+      from_elasticsearch
+    else
+      from_database
+    end
+  end
+
+  private
+
+  def from_elasticsearch
+    query = {
+      function_score: {
+        query: {
+          multi_match: {
+            query: @query,
+            fields: %w(name.edge_ngram name),
+            type: 'most_fields',
+            operator: 'and',
+          },
+        },
+
+        functions: [
+          {
+            field_value_factor: {
+              field: 'usage',
+              modifier: 'log2p',
+              missing: 0,
+            },
+          },
+
+          {
+            gauss: {
+              last_status_at: {
+                scale: '7d',
+                offset: '14d',
+                decay: 0.5,
+              },
+            },
+          },
+        ],
+
+        boost_mode: 'multiply',
+      },
+    }
+
+    filter = {
+      bool: {
+        should: [
+          {
+            term: {
+              reviewed: {
+                value: true,
+              },
+            },
+          },
+
+          {
+            term: {
+              name: {
+                value: @query,
+              },
+            },
+          },
+        ],
+      },
+    }
+
+    TagsIndex.query(query).filter(filter).limit(@limit).offset(@offset).objects.compact
+  end
+
+  def from_database
+    Tag.search_for(@query, @limit, @offset)
+  end
+end
diff --git a/config/locales/simple_form.en.yml b/config/locales/simple_form.en.yml
index e15d5904f..98f0843d0 100644
--- a/config/locales/simple_form.en.yml
+++ b/config/locales/simple_form.en.yml
@@ -142,7 +142,7 @@ en:
         report: Send e-mail when a new report is submitted
         trending_tag: Send e-mail when an unreviewed hashtag is trending
       tag:
-        listable: Allow this hashtag to appear on the profile directory
+        listable: Allow this hashtag to appear in searches and on the profile directory
         trendable: Allow this hashtag to appear under trends
         usable: Allow toots to use this hashtag
     'no': 'No'
diff --git a/db/migrate/20190815225426_add_last_status_at_to_tags.rb b/db/migrate/20190815225426_add_last_status_at_to_tags.rb
new file mode 100644
index 000000000..d83537c47
--- /dev/null
+++ b/db/migrate/20190815225426_add_last_status_at_to_tags.rb
@@ -0,0 +1,6 @@
+class AddLastStatusAtToTags < ActiveRecord::Migration[5.2]
+  def change
+    add_column :tags, :last_status_at, :datetime
+    add_column :tags, :last_trend_at, :datetime
+  end
+end
diff --git a/db/schema.rb b/db/schema.rb
index f8fc6a821..0da20d62f 100644
--- a/db/schema.rb
+++ b/db/schema.rb
@@ -10,7 +10,7 @@
 #
 # It's strongly recommended that you check this file into your version control system.
 
-ActiveRecord::Schema.define(version: 2019_08_07_135426) do
+ActiveRecord::Schema.define(version: 2019_08_15_225426) do
 
   # These are extensions that must be enabled in order to support this database
   enable_extension "plpgsql"
@@ -667,6 +667,8 @@ ActiveRecord::Schema.define(version: 2019_08_07_135426) do
     t.boolean "listable"
     t.datetime "reviewed_at"
     t.datetime "requested_review_at"
+    t.datetime "last_status_at"
+    t.datetime "last_trend_at"
     t.index "lower((name)::text)", name: "index_tags_on_name_lower", unique: true
   end
 
diff --git a/spec/models/tag_spec.rb b/spec/models/tag_spec.rb
index 9d700849b..2bb30fb57 100644
--- a/spec/models/tag_spec.rb
+++ b/spec/models/tag_spec.rb
@@ -136,8 +136,8 @@ RSpec.describe Tag, type: :model do
     end
 
     it 'finds the exact matching tag as the first item' do
-      similar_tag = Fabricate(:tag, name: "matchlater", score: 1)
-      tag = Fabricate(:tag, name: "match", score: 1)
+      similar_tag = Fabricate(:tag, name: "matchlater", reviewed_at: Time.now.utc)
+      tag = Fabricate(:tag, name: "match", reviewed_at: Time.now.utc)
 
       results = Tag.search_for("match")
 
-- 
cgit 


From 96702e7f6727d9c688b2c6e2527c4a5bfefff886 Mon Sep 17 00:00:00 2001
From: Eugen Rochko <eugen@zeonfederated.com>
Date: Sun, 18 Aug 2019 14:55:03 +0200
Subject: Add `tootctl cache recount` command (#11597)

---
 app/models/concerns/account_counters.rb |  3 ++-
 app/models/status.rb                    |  7 +++--
 lib/mastodon/cache_cli.rb               | 45 +++++++++++++++++++++++++++++++++
 3 files changed, 52 insertions(+), 3 deletions(-)

(limited to 'app/models')

diff --git a/app/models/concerns/account_counters.rb b/app/models/concerns/account_counters.rb
index 3581df8dd..6e25e1905 100644
--- a/app/models/concerns/account_counters.rb
+++ b/app/models/concerns/account_counters.rb
@@ -26,7 +26,8 @@ module AccountCounters
   private
 
   def save_account_stat
-    return unless account_stat&.changed?
+    return unless association(:account_stat).loaded? && account_stat&.changed?
+
     account_stat.save
   end
 end
diff --git a/app/models/status.rb b/app/models/status.rb
index 23682c84b..0538c4e9e 100644
--- a/app/models/status.rb
+++ b/app/models/status.rb
@@ -388,13 +388,16 @@ class Status < ApplicationRecord
     end
   end
 
+  def status_stat
+    super || build_status_stat
+  end
+
   private
 
   def update_status_stat!(attrs)
     return if marked_for_destruction? || destroyed?
 
-    record = status_stat || build_status_stat
-    record.update(attrs)
+    status_stat.update(attrs)
   end
 
   def store_uri
diff --git a/lib/mastodon/cache_cli.rb b/lib/mastodon/cache_cli.rb
index e9b6667b3..5b0eea91b 100644
--- a/lib/mastodon/cache_cli.rb
+++ b/lib/mastodon/cache_cli.rb
@@ -15,5 +15,50 @@ module Mastodon
       Rails.cache.clear
       say('OK', :green)
     end
+
+    desc 'recount TYPE', 'Update hard-cached counters'
+    long_desc <<~LONG_DESC
+      Update hard-cached counters of TYPE by counting referenced
+      records from scratch. TYPE can be "accounts" or "statuses".
+
+      It may take a very long time to finish, depending on the
+      size of the database.
+    LONG_DESC
+    def recount(type)
+      processed = 0
+
+      case type
+      when 'accounts'
+        Account.local.includes(:account_stat).find_each do |account|
+          account_stat                 = account.account_stat
+          account_stat.following_count = account.active_relationships.count
+          account_stat.followers_count = account.passive_relationships.count
+          account_stat.statuses_count  = account.statuses.where.not(visibility: :direct).count
+
+          account_stat.save if account_stat.changed?
+
+          processed += 1
+          say('.', :green, false)
+        end
+      when 'statuses'
+        Status.includes(:status_stat).find_each do |status|
+          status_stat                  = status.status_stat
+          status_stat.replies_count    = status.replies.where.not(visibility: :direct).count
+          status_stat.reblogs_count    = status.reblogs.count
+          status_stat.favourites_count = status.favourites.count
+
+          status_stat.save if status_stat.changed?
+
+          processed += 1
+          say('.', :green, false)
+        end
+      else
+        say("Unknown type: #{type}", :red)
+        exit(1)
+      end
+
+      say
+      say("OK, recounted #{processed} records", :green)
+    end
   end
 end
-- 
cgit 


From 9b6a5ed109e1986149c1f15a41d4f442ae8ae39c Mon Sep 17 00:00:00 2001
From: ThibG <thib@sitedethib.com>
Date: Mon, 19 Aug 2019 11:35:48 +0200
Subject: Add public blocks to /about/blocks (#11298)
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

* Add automatic blocklist display in /about/blocks

Inspired by https://github.com/Gargron/mastodon.social-misc

* Add admin option to set who can see instance blocks

* Normalize locales files

* Rename “Sandbox” to “Silence” for consistency

* Disable /about/blocks when in whitelist mode

* Optionally display rationale for domain blocks

* Only display domain blocks that have user-facing limitations, and order them

* Redesign table of blocked domains to better handle long domain names and rationales

* Change domain blocks ordering now that rationales aren't displayed right away

* Only show explanation for block severities actually in use

* Reword instance block explanations and add disclaimer for public fetch mode
---
 app/controllers/about_controller.rb        | 34 ++++++++++++++-
 app/javascript/packs/public.js             |  9 ++++
 app/javascript/styles/mastodon/tables.scss | 67 ++++++++++++++++++++++++++++++
 app/models/domain_block.rb                 |  1 +
 app/models/form/admin_settings.rb          |  4 ++
 app/views/about/blocks.html.haml           | 48 +++++++++++++++++++++
 app/views/admin/settings/edit.html.haml    |  6 +++
 config/locales/en.yml                      | 24 +++++++++++
 config/routes.rb                           |  7 ++--
 config/settings.yml                        |  2 +
 10 files changed, 197 insertions(+), 5 deletions(-)
 create mode 100644 app/views/about/blocks.html.haml

(limited to 'app/models')

diff --git a/app/controllers/about_controller.rb b/app/controllers/about_controller.rb
index d276e8fe5..5e942e5c0 100644
--- a/app/controllers/about_controller.rb
+++ b/app/controllers/about_controller.rb
@@ -3,10 +3,12 @@
 class AboutController < ApplicationController
   layout 'public'
 
-  before_action :require_open_federation!, only: [:show, :more]
+  before_action :require_open_federation!, only: [:show, :more, :blocks]
+  before_action :check_blocklist_enabled, only: [:blocks]
+  before_action :authenticate_user!, only: [:blocks], if: :blocklist_account_required?
   before_action :set_body_classes, only: :show
   before_action :set_instance_presenter
-  before_action :set_expires_in
+  before_action :set_expires_in, only: [:show, :more, :terms]
 
   skip_before_action :require_functional!, only: [:more, :terms]
 
@@ -18,12 +20,40 @@ class AboutController < ApplicationController
 
   def terms; end
 
+  def blocks
+    @show_rationale = Setting.show_domain_blocks_rationale == 'all'
+    @show_rationale |= Setting.show_domain_blocks_rationale == 'users' && !current_user.nil? && current_user.functional?
+    @blocks = DomainBlock.with_user_facing_limitations.order('(CASE severity WHEN 0 THEN 1 WHEN 1 THEN 2 WHEN 2 THEN 0 END), reject_media, domain').to_a
+  end
+
   private
 
   def require_open_federation!
     not_found if whitelist_mode?
   end
 
+  def check_blocklist_enabled
+    not_found if Setting.show_domain_blocks == 'disabled'
+  end
+
+  def blocklist_account_required?
+    Setting.show_domain_blocks == 'users'
+  end
+
+  def block_severity_text(block)
+    if block.severity == 'suspend'
+      I18n.t('domain_blocks.suspension')
+    else
+      limitations = []
+      limitations << I18n.t('domain_blocks.media_block') if block.reject_media?
+      limitations << I18n.t('domain_blocks.silence') if block.severity == 'silence'
+      limitations.join(', ')
+    end
+  end
+
+  helper_method :block_severity_text
+  helper_method :public_fetch_mode?
+
   def new_user
     User.new.tap do |user|
       user.build_account
diff --git a/app/javascript/packs/public.js b/app/javascript/packs/public.js
index b58622a8d..c5cd7129f 100644
--- a/app/javascript/packs/public.js
+++ b/app/javascript/packs/public.js
@@ -141,6 +141,15 @@ function main() {
     return false;
   });
 
+  delegate(document, '.blocks-table button.icon-button', 'click', function(e) {
+    e.preventDefault();
+
+    const classList = this.firstElementChild.classList;
+    classList.toggle('fa-chevron-down');
+    classList.toggle('fa-chevron-up');
+    this.parentElement.parentElement.nextElementSibling.classList.toggle('hidden');
+  });
+
   delegate(document, '.modal-button', 'click', e => {
     e.preventDefault();
 
diff --git a/app/javascript/styles/mastodon/tables.scss b/app/javascript/styles/mastodon/tables.scss
index 11ac6dfeb..fe6beba5d 100644
--- a/app/javascript/styles/mastodon/tables.scss
+++ b/app/javascript/styles/mastodon/tables.scss
@@ -241,3 +241,70 @@ a.table-action-link {
     }
   }
 }
+
+.blocks-table {
+  width: 100%;
+  max-width: 100%;
+  border-spacing: 0;
+  border-collapse: collapse;
+  table-layout: fixed;
+  border: 1px solid darken($ui-base-color, 8%);
+
+  thead {
+    border: 1px solid darken($ui-base-color, 8%);
+    background: darken($ui-base-color, 4%);
+    font-weight: 500;
+
+    th.severity-column {
+      width: 120px;
+    }
+
+    th.button-column {
+      width: 23px;
+    }
+  }
+
+  tbody > tr {
+    border: 1px solid darken($ui-base-color, 8%);
+    border-bottom: 0;
+    background: darken($ui-base-color, 4%);
+
+    &:hover {
+      background: darken($ui-base-color, 2%);
+    }
+
+    &.even {
+      background: $ui-base-color;
+
+      &:hover {
+        background: lighten($ui-base-color, 2%);
+      }
+    }
+
+    &.rationale {
+      background: lighten($ui-base-color, 4%);
+      border-top: 0;
+
+      &:hover {
+        background: lighten($ui-base-color, 6%);
+      }
+
+      &.hidden {
+        display: none;
+      }
+    }
+
+    td:first-child {
+      overflow: hidden;
+      text-overflow: ellipsis;
+    }
+  }
+
+  th,
+  td {
+    padding: 8px;
+    line-height: 18px;
+    vertical-align: top;
+    text-align: left;
+  }
+}
diff --git a/app/models/domain_block.rb b/app/models/domain_block.rb
index 37b8d98c6..4383cbd05 100644
--- a/app/models/domain_block.rb
+++ b/app/models/domain_block.rb
@@ -25,6 +25,7 @@ class DomainBlock < ApplicationRecord
   delegate :count, to: :accounts, prefix: true
 
   scope :matches_domain, ->(value) { where(arel_table[:domain].matches("%#{value}%")) }
+  scope :with_user_facing_limitations, -> { where(severity: [:silence, :suspend]).or(where(reject_media: true)) }
 
   class << self
     def suspend?(domain)
diff --git a/app/models/form/admin_settings.rb b/app/models/form/admin_settings.rb
index 051268375..6bc3ca9f5 100644
--- a/app/models/form/admin_settings.rb
+++ b/app/models/form/admin_settings.rb
@@ -30,6 +30,8 @@ class Form::AdminSettings
     mascot
     spam_check_enabled
     trends
+    show_domain_blocks
+    show_domain_blocks_rationale
   ).freeze
 
   BOOLEAN_KEYS = %i(
@@ -60,6 +62,8 @@ class Form::AdminSettings
   validates :site_contact_email, :site_contact_username, presence: true
   validates :site_contact_username, existing_username: true
   validates :bootstrap_timeline_accounts, existing_username: { multiple: true }
+  validates :show_domain_blocks, inclusion: { in: %w(disabled users all) }
+  validates :show_domain_blocks_rationale, inclusion: { in: %w(disabled users all) }
 
   def initialize(_attributes = {})
     super
diff --git a/app/views/about/blocks.html.haml b/app/views/about/blocks.html.haml
new file mode 100644
index 000000000..a81a4d1eb
--- /dev/null
+++ b/app/views/about/blocks.html.haml
@@ -0,0 +1,48 @@
+- content_for :page_title do
+  = t('domain_blocks.title', instance: site_hostname)
+
+.grid
+  .column-0
+    .box-widget.rich-formatting
+      %h2= t('domain_blocks.blocked_domains')
+      %p= t('domain_blocks.description', instance: site_hostname)
+      .table-wrapper
+        %table.blocks-table
+          %thead
+            %tr
+              %th= t('domain_blocks.domain')
+              %th.severity-column= t('domain_blocks.severity')
+              - if @show_rationale
+                %th.button-column
+          %tbody
+            - if @blocks.empty?
+              %tr
+                %td{ colspan: @show_rationale ? 3 : 2 }= t('domain_blocks.no_domain_blocks')
+            - else
+              - @blocks.each_with_index do |block, i|
+                %tr{ class: i % 2 == 0 ? 'even': nil }
+                  %td{ title: block.domain }= block.domain
+                  %td= block_severity_text(block)
+                  - if @show_rationale
+                    %td
+                      - if block.public_comment.present?
+                        %button.icon-button{ title: t('domain_blocks.show_rationale'), 'aria-label' => t('domain_blocks.show_rationale') }
+                          = fa_icon 'chevron-down fw', 'aria-hidden' => true
+                - if @show_rationale
+                  - if block.public_comment.present?
+                    %tr.rationale.hidden
+                      %td{ colspan: 3 }= block.public_comment.presence
+      %h2= t('domain_blocks.severity_legend.title')
+      - if @blocks.any? { |block| block.reject_media? }
+        %h3= t('domain_blocks.media_block')
+        %p= t('domain_blocks.severity_legend.media_block')
+      - if @blocks.any? { |block| block.severity == 'silence' }
+        %h3= t('domain_blocks.silence')
+        %p= t('domain_blocks.severity_legend.silence')
+      - if @blocks.any? { |block| block.severity == 'suspend' }
+        %h3= t('domain_blocks.suspension')
+        %p= t('domain_blocks.severity_legend.suspension')
+        - if public_fetch_mode?
+          %p= t('domain_blocks.severity_legend.suspension_disclaimer')
+  .column-1
+    = render 'application/sidebar'
diff --git a/app/views/admin/settings/edit.html.haml b/app/views/admin/settings/edit.html.haml
index 28c0ece15..28880c087 100644
--- a/app/views/admin/settings/edit.html.haml
+++ b/app/views/admin/settings/edit.html.haml
@@ -79,6 +79,12 @@
   .fields-group
     = f.input :min_invite_role, wrapper: :with_label, collection: %i(disabled user moderator admin), label: t('admin.settings.registrations.min_invite_role.title'), label_method: lambda { |role| role == :disabled ? t('admin.settings.registrations.min_invite_role.disabled') : t("admin.accounts.roles.#{role}") }, include_blank: false, collection_wrapper_tag: 'ul', item_wrapper_tag: 'li'
 
+  .fields-row
+    .fields-row__column.fields-row__column-6.fields-group
+      = f.input :show_domain_blocks, wrapper: :with_label, collection: %i(disabled users all), label: t('admin.settings.domain_blocks.title'), label_method: lambda { |value| t("admin.settings.domain_blocks.#{value}") }, include_blank: false, collection_wrapper_tag: 'ul', item_wrapper_tag: 'li'
+    .fields-row__column.fields-row__column-6.fields-group
+      = f.input :show_domain_blocks_rationale, wrapper: :with_label, collection: %i(disabled users all), label: t('admin.settings.domain_blocks_rationale.title'), label_method: lambda { |value| t("admin.settings.domain_blocks.#{value}") }, include_blank: false, collection_wrapper_tag: 'ul', item_wrapper_tag: 'li'
+
   .fields-group
     = f.input :closed_registrations_message, as: :text, wrapper: :with_block_label, label: t('admin.settings.registrations.closed_message.title'), hint: t('admin.settings.registrations.closed_message.desc_html'), input_html: { rows: 8 }
     = f.input :site_extended_description, wrapper: :with_block_label, as: :text, label: t('admin.settings.site_description_extended.title'), hint: t('admin.settings.site_description_extended.desc_html'), input_html: { rows: 8 } unless whitelist_mode?
diff --git a/config/locales/en.yml b/config/locales/en.yml
index 4696dc11b..8d267065c 100644
--- a/config/locales/en.yml
+++ b/config/locales/en.yml
@@ -423,6 +423,13 @@ en:
       custom_css:
         desc_html: Modify the look with CSS loaded on every page
         title: Custom CSS
+      domain_blocks:
+        all: To everyone
+        disabled: To no one
+        title: Show domain blocks
+        users: To logged-in local users
+      domain_blocks_rationale:
+        title: Show rationale
       hero:
         desc_html: Displayed on the frontpage. At least 600x100px recommended. When not set, falls back to server thumbnail
         title: Hero image
@@ -630,6 +637,23 @@ en:
     people:
       one: "%{count} person"
       other: "%{count} people"
+  domain_blocks:
+    blocked_domains: List of limited and blocked domains
+    description: This is the list of servers that %{instance} limits or reject federation with.
+    domain: Domain
+    media_block: Media block
+    no_domain_blocks: "(No domain blocks)"
+    severity: Severity
+    severity_legend:
+      media_block: Media files coming from the server are neither fetched, stored, or displayed to the user.
+      silence: Accounts from silenced servers can be found, followed and interacted with, but their toots will not appear in the public timelines, and notifications from them will not reach local users who are not following them.
+      suspension: No content from suspended servers is stored or displayed, nor is any content sent to them. Interactions from suspended servers are ignored.
+      suspension_disclaimer: Suspended servers may occasionally retrieve public content from this server.
+      title: Severities
+    show_rationale: Show rationale
+    silence: Silence
+    suspension: Suspension
+    title: "%{instance} List of blocked instances"
   domain_validator:
     invalid_domain: is not a valid domain name
   errors:
diff --git a/config/routes.rb b/config/routes.rb
index 9c33b8190..9ae24b0cd 100644
--- a/config/routes.rb
+++ b/config/routes.rb
@@ -423,9 +423,10 @@ Rails.application.routes.draw do
 
   get '/web/(*any)', to: 'home#index', as: :web
 
-  get '/about',      to: 'about#show'
-  get '/about/more', to: 'about#more'
-  get '/terms',      to: 'about#terms'
+  get '/about',        to: 'about#show'
+  get '/about/more',   to: 'about#more'
+  get '/about/blocks', to: 'about#blocks'
+  get '/terms',        to: 'about#terms'
 
   root 'home#index'
 
diff --git a/config/settings.yml b/config/settings.yml
index 4e5eefb59..6dbc46706 100644
--- a/config/settings.yml
+++ b/config/settings.yml
@@ -64,6 +64,8 @@ defaults: &defaults
   peers_api_enabled: true
   show_known_fediverse_at_about_page: true
   spam_check_enabled: true
+  show_domain_blocks: 'disabled'
+  show_domain_blocks_rationale: 'disabled'
 
 development:
   <<: *defaults
-- 
cgit 


From cb62a83a71a9679061f8daa468119124b5da5e76 Mon Sep 17 00:00:00 2001
From: ThibG <thib@sitedethib.com>
Date: Mon, 19 Aug 2019 11:40:42 +0200
Subject: Add invite comments (#10465)

---
 app/controllers/invites_controller.rb               | 2 +-
 app/models/invite.rb                                | 3 +++
 app/views/invites/_form.html.haml                   | 3 +++
 app/views/invites/_invite.html.haml                 | 3 +++
 app/views/invites/index.html.haml                   | 1 +
 db/migrate/20190403141604_add_comment_to_invites.rb | 5 +++++
 db/schema.rb                                        | 1 +
 7 files changed, 17 insertions(+), 1 deletion(-)
 create mode 100644 db/migrate/20190403141604_add_comment_to_invites.rb

(limited to 'app/models')

diff --git a/app/controllers/invites_controller.rb b/app/controllers/invites_controller.rb
index de5280305..8d92147e2 100644
--- a/app/controllers/invites_controller.rb
+++ b/app/controllers/invites_controller.rb
@@ -43,7 +43,7 @@ class InvitesController < ApplicationController
   end
 
   def resource_params
-    params.require(:invite).permit(:max_uses, :expires_in, :autofollow)
+    params.require(:invite).permit(:max_uses, :expires_in, :autofollow, :comment)
   end
 
   def set_body_classes
diff --git a/app/models/invite.rb b/app/models/invite.rb
index 02ab8e0b2..29d25eae8 100644
--- a/app/models/invite.rb
+++ b/app/models/invite.rb
@@ -12,6 +12,7 @@
 #  created_at :datetime         not null
 #  updated_at :datetime         not null
 #  autofollow :boolean          default(FALSE), not null
+#  comment    :text
 #
 
 class Invite < ApplicationRecord
@@ -22,6 +23,8 @@ class Invite < ApplicationRecord
 
   scope :available, -> { where(expires_at: nil).or(where('expires_at >= ?', Time.now.utc)) }
 
+  validates :comment, length: { maximum: 420 }
+
   before_validation :set_code
 
   def valid_for_use?
diff --git a/app/views/invites/_form.html.haml b/app/views/invites/_form.html.haml
index 3a2a5ef0e..b19f70539 100644
--- a/app/views/invites/_form.html.haml
+++ b/app/views/invites/_form.html.haml
@@ -10,5 +10,8 @@
   .fields-group
     = f.input :autofollow, wrapper: :with_label
 
+  .fields-group
+    = f.input :comment, wrapper: :with_label, input_html: { maxlength: 420 }
+
   .actions
     = f.button :button, t('invites.generate'), type: :submit
diff --git a/app/views/invites/_invite.html.haml b/app/views/invites/_invite.html.haml
index 62799ca5b..03050c868 100644
--- a/app/views/invites/_invite.html.haml
+++ b/app/views/invites/_invite.html.haml
@@ -20,6 +20,9 @@
     %td{ colspan: 2 }
       = t('invites.expired')
 
+  %td
+    = invite.comment
+
   %td
     - if invite.valid_for_use? && policy(invite).destroy?
       = table_link_to 'times', t('invites.delete'), invite_path(invite), method: :delete
diff --git a/app/views/invites/index.html.haml b/app/views/invites/index.html.haml
index 61420ab1e..62065d6ae 100644
--- a/app/views/invites/index.html.haml
+++ b/app/views/invites/index.html.haml
@@ -15,6 +15,7 @@
         %th
         %th= t('invites.table.uses')
         %th= t('invites.table.expires_at')
+        %th= t('invites.table.comment')
         %th
     %tbody
       = render @invites
diff --git a/db/migrate/20190403141604_add_comment_to_invites.rb b/db/migrate/20190403141604_add_comment_to_invites.rb
new file mode 100644
index 000000000..f0d7b1dcd
--- /dev/null
+++ b/db/migrate/20190403141604_add_comment_to_invites.rb
@@ -0,0 +1,5 @@
+class AddCommentToInvites < ActiveRecord::Migration[5.2]
+  def change
+    add_column :invites, :comment, :text
+  end
+end
diff --git a/db/schema.rb b/db/schema.rb
index 0da20d62f..18f615d61 100644
--- a/db/schema.rb
+++ b/db/schema.rb
@@ -344,6 +344,7 @@ ActiveRecord::Schema.define(version: 2019_08_15_225426) do
     t.datetime "created_at", null: false
     t.datetime "updated_at", null: false
     t.boolean "autofollow", default: false, null: false
+    t.text "comment"
     t.index ["code"], name: "index_invites_on_code", unique: true
     t.index ["user_id"], name: "index_invites_on_user_id"
   end
-- 
cgit 


From dff46b260b2f7d765d254c84a4b89105c7de5e97 Mon Sep 17 00:00:00 2001
From: Eugen Rochko <eugen@zeonfederated.com>
Date: Mon, 19 Aug 2019 20:36:44 +0200
Subject: Fix ignoring whole status because of one invalid hashtag (#11621)

Fix #11618
---
 app/models/tag.rb | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

(limited to 'app/models')

diff --git a/app/models/tag.rb b/app/models/tag.rb
index 5094d973d..945e3a3c6 100644
--- a/app/models/tag.rb
+++ b/app/models/tag.rb
@@ -114,7 +114,7 @@ class Tag < ApplicationRecord
   class << self
     def find_or_create_by_names(name_or_names)
       Array(name_or_names).map(&method(:normalize)).uniq { |str| str.mb_chars.downcase.to_s }.map do |normalized_name|
-        tag = matching_name(normalized_name).first || create(name: normalized_name)
+        tag = matching_name(normalized_name).first || create!(name: normalized_name)
 
         yield tag if block_given?
 
-- 
cgit 


From 97192d9a77c0b4b68afe50d6a94d87110a8adbcd Mon Sep 17 00:00:00 2001
From: Eugen Rochko <eugen@zeonfederated.com>
Date: Thu, 22 Aug 2019 04:17:12 +0200
Subject: Fix remote and staff-removed statuses leaving media behind for a day
 (#11638)

The reason for unattaching media instead of removing it is to support
delete & redraft functionality, but remote or staff-removed statuses
will never be redrafted, so the media should be deleted immediately
---
 app/controllers/api/v1/statuses_controller.rb               |  2 +-
 app/lib/activitypub/activity/delete.rb                      |  2 +-
 app/models/form/status_batch.rb                             |  2 +-
 app/services/batched_remove_status_service.rb               |  2 +-
 app/services/remove_status_service.rb                       | 12 ++++++++++++
 app/workers/removal_worker.rb                               |  4 ++--
 spec/controllers/admin/reported_statuses_controller_spec.rb |  2 +-
 spec/controllers/admin/statuses_controller_spec.rb          |  2 +-
 spec/models/form/status_batch_spec.rb                       |  4 ++--
 9 files changed, 22 insertions(+), 10 deletions(-)

(limited to 'app/models')

diff --git a/app/controllers/api/v1/statuses_controller.rb b/app/controllers/api/v1/statuses_controller.rb
index 71a505c26..39ca56482 100644
--- a/app/controllers/api/v1/statuses_controller.rb
+++ b/app/controllers/api/v1/statuses_controller.rb
@@ -53,7 +53,7 @@ class Api::V1::StatusesController < Api::BaseController
     @status = Status.where(account_id: current_user.account).find(params[:id])
     authorize @status, :destroy?
 
-    RemovalWorker.perform_async(@status.id)
+    RemovalWorker.perform_async(@status.id, redraft: true)
 
     render json: @status, serializer: REST::StatusSerializer, source_requested: true
   end
diff --git a/app/lib/activitypub/activity/delete.rb b/app/lib/activitypub/activity/delete.rb
index 1f2b40c15..345060462 100644
--- a/app/lib/activitypub/activity/delete.rb
+++ b/app/lib/activitypub/activity/delete.rb
@@ -70,7 +70,7 @@ class ActivityPub::Activity::Delete < ActivityPub::Activity
   end
 
   def delete_now!
-    RemoveStatusService.new.call(@status)
+    RemoveStatusService.new.call(@status, redraft: false)
   end
 
   def payload
diff --git a/app/models/form/status_batch.rb b/app/models/form/status_batch.rb
index 933dfdaca..831d8b7c5 100644
--- a/app/models/form/status_batch.rb
+++ b/app/models/form/status_batch.rb
@@ -34,7 +34,7 @@ class Form::StatusBatch
 
   def delete_statuses
     Status.where(id: status_ids).reorder(nil).find_each do |status|
-      RemovalWorker.perform_async(status.id)
+      RemovalWorker.perform_async(status.id, redraft: false)
       Tombstone.find_or_create_by(uri: status.uri, account: status.account, by_moderator: true)
       log_action :destroy, status
     end
diff --git a/app/services/batched_remove_status_service.rb b/app/services/batched_remove_status_service.rb
index 6df8d4769..3638134be 100644
--- a/app/services/batched_remove_status_service.rb
+++ b/app/services/batched_remove_status_service.rb
@@ -8,7 +8,7 @@ class BatchedRemoveStatusService < BaseService
   # Dispatch Salmon deletes, unique per domain, of the deleted statuses, but only local ones
   # Remove statuses from home feeds
   # Push delete events to streaming API for home feeds and public feeds
-  # @param [Status] statuses A preferably batched array of statuses
+  # @param [Enumerable<Status>] statuses A preferably batched array of statuses
   # @param [Hash] options
   # @option [Boolean] :skip_side_effects
   def call(statuses, **options)
diff --git a/app/services/remove_status_service.rb b/app/services/remove_status_service.rb
index 91c934181..685c1d4bf 100644
--- a/app/services/remove_status_service.rb
+++ b/app/services/remove_status_service.rb
@@ -4,6 +4,11 @@ class RemoveStatusService < BaseService
   include Redisable
   include Payloadable
 
+  # Delete a status
+  # @param   [Status] status
+  # @param   [Hash] options
+  # @option  [Boolean] :redraft
+  # @options [Boolean] :original_removed
   def call(status, **options)
     @payload  = Oj.dump(event: :delete, payload: status.id.to_s)
     @status   = status
@@ -24,6 +29,7 @@ class RemoveStatusService < BaseService
         remove_from_public
         remove_from_media if status.media_attachments.any?
         remove_from_spam_check
+        remove_media
 
         @status.destroy!
       else
@@ -143,6 +149,12 @@ class RemoveStatusService < BaseService
     redis.publish('timeline:public:local:media', @payload) if @status.local?
   end
 
+  def remove_media
+    return if @options[:redraft]
+
+    @status.media_attachments.destroy_all
+  end
+
   def remove_from_spam_check
     redis.zremrangebyscore("spam_check:#{@status.account_id}", @status.id, @status.id)
   end
diff --git a/app/workers/removal_worker.rb b/app/workers/removal_worker.rb
index 19a660dd3..14423a4fb 100644
--- a/app/workers/removal_worker.rb
+++ b/app/workers/removal_worker.rb
@@ -3,8 +3,8 @@
 class RemovalWorker
   include Sidekiq::Worker
 
-  def perform(status_id)
-    RemoveStatusService.new.call(Status.find(status_id))
+  def perform(status_id, options = {})
+    RemoveStatusService.new.call(Status.find(status_id), **options.symbolize_keys)
   rescue ActiveRecord::RecordNotFound
     true
   end
diff --git a/spec/controllers/admin/reported_statuses_controller_spec.rb b/spec/controllers/admin/reported_statuses_controller_spec.rb
index c358506d6..bd146b795 100644
--- a/spec/controllers/admin/reported_statuses_controller_spec.rb
+++ b/spec/controllers/admin/reported_statuses_controller_spec.rb
@@ -47,7 +47,7 @@ describe Admin::ReportedStatusesController do
       it 'removes a status' do
         allow(RemovalWorker).to receive(:perform_async)
         subject.call
-        expect(RemovalWorker).to have_received(:perform_async).with(status_ids.first)
+        expect(RemovalWorker).to have_received(:perform_async).with(status_ids.first, redraft: false)
       end
     end
 
diff --git a/spec/controllers/admin/statuses_controller_spec.rb b/spec/controllers/admin/statuses_controller_spec.rb
index 1a08c10b7..6b06343ef 100644
--- a/spec/controllers/admin/statuses_controller_spec.rb
+++ b/spec/controllers/admin/statuses_controller_spec.rb
@@ -65,7 +65,7 @@ describe Admin::StatusesController do
       it 'removes a status' do
         allow(RemovalWorker).to receive(:perform_async)
         subject.call
-        expect(RemovalWorker).to have_received(:perform_async).with(status_ids.first)
+        expect(RemovalWorker).to have_received(:perform_async).with(status_ids.first, redraft: false)
       end
     end
 
diff --git a/spec/models/form/status_batch_spec.rb b/spec/models/form/status_batch_spec.rb
index 00c790a11..f9c58c90f 100644
--- a/spec/models/form/status_batch_spec.rb
+++ b/spec/models/form/status_batch_spec.rb
@@ -41,12 +41,12 @@ describe Form::StatusBatch do
 
     it 'call RemovalWorker' do
       form.save
-      expect(RemovalWorker).to have_received(:perform_async).with(status.id)
+      expect(RemovalWorker).to have_received(:perform_async).with(status.id, redraft: false)
     end
 
     it 'do not call RemovalWorker' do
       form.save
-      expect(RemovalWorker).not_to have_received(:perform_async).with(another_status.id)
+      expect(RemovalWorker).not_to have_received(:perform_async).with(another_status.id, redraft: false)
     end
   end
 end
-- 
cgit 


From 282ea170782e4ce1ed5251a1b94857a512412397 Mon Sep 17 00:00:00 2001
From: Eugen Rochko <eugen@zeonfederated.com>
Date: Thu, 22 Aug 2019 21:55:56 +0200
Subject: Add soft delete for statuses for instant deletes through API (#11623)

* Add soft delete for statuses to allow them to appear instant

* Allow reporting soft-deleted statuses and show them in the admin UI

* Change index for getting an account's statuses
---
 Gemfile                                                 |  1 +
 Gemfile.lock                                            |  3 +++
 app/controllers/api/v1/reports_controller.rb            |  2 +-
 app/controllers/api/v1/statuses/reblogs_controller.rb   |  3 ++-
 app/controllers/api/v1/statuses_controller.rb           |  1 +
 app/models/form/status_batch.rb                         |  1 +
 app/models/report.rb                                    |  2 +-
 app/models/status.rb                                    |  6 +++++-
 app/views/admin/reports/_status.html.haml               |  5 ++++-
 app/workers/removal_worker.rb                           |  2 +-
 config/locales/en.yml                                   |  1 +
 db/migrate/20190819134503_add_deleted_at_to_statuses.rb |  5 +++++
 db/migrate/20190820003045_update_statuses_index.rb      | 13 +++++++++++++
 db/schema.rb                                            |  5 +++--
 14 files changed, 42 insertions(+), 8 deletions(-)
 create mode 100644 db/migrate/20190819134503_add_deleted_at_to_statuses.rb
 create mode 100644 db/migrate/20190820003045_update_statuses_index.rb

(limited to 'app/models')

diff --git a/Gemfile b/Gemfile
index 250a28a3a..86dab965a 100644
--- a/Gemfile
+++ b/Gemfile
@@ -43,6 +43,7 @@ gem 'omniauth-cas', '~> 1.1'
 gem 'omniauth-saml', '~> 1.10'
 gem 'omniauth', '~> 1.9'
 
+gem 'discard', '~> 1.1'
 gem 'doorkeeper', '~> 5.1'
 gem 'fast_blank', '~> 1.0'
 gem 'fastimage'
diff --git a/Gemfile.lock b/Gemfile.lock
index 1da6d73a6..b896909a4 100644
--- a/Gemfile.lock
+++ b/Gemfile.lock
@@ -204,6 +204,8 @@ GEM
       devise (>= 4.0.0)
       rpam2 (~> 4.0)
     diff-lcs (1.3)
+    discard (1.1.0)
+      activerecord (>= 4.2, < 7)
     docile (1.3.2)
     domain_name (0.5.20180417)
       unf (>= 0.0.5, < 1.0.0)
@@ -692,6 +694,7 @@ DEPENDENCIES
   devise (~> 4.6)
   devise-two-factor (~> 3.1)
   devise_pam_authenticatable2 (~> 9.2)
+  discard (~> 1.1)
   doorkeeper (~> 5.1)
   dotenv-rails (~> 2.7)
   fabrication (~> 2.20)
diff --git a/app/controllers/api/v1/reports_controller.rb b/app/controllers/api/v1/reports_controller.rb
index e182a9c6c..1b0b4b05b 100644
--- a/app/controllers/api/v1/reports_controller.rb
+++ b/app/controllers/api/v1/reports_controller.rb
@@ -21,7 +21,7 @@ class Api::V1::ReportsController < Api::BaseController
   private
 
   def reported_status_ids
-    reported_account.statuses.find(status_ids).pluck(:id)
+    reported_account.statuses.with_discarded.find(status_ids).pluck(:id)
   end
 
   def status_ids
diff --git a/app/controllers/api/v1/statuses/reblogs_controller.rb b/app/controllers/api/v1/statuses/reblogs_controller.rb
index ed4f55100..42381a37f 100644
--- a/app/controllers/api/v1/statuses/reblogs_controller.rb
+++ b/app/controllers/api/v1/statuses/reblogs_controller.rb
@@ -18,6 +18,7 @@ class Api::V1::Statuses::ReblogsController < Api::BaseController
     @reblogs_map = { @status.id => false }
 
     authorize status_for_destroy, :unreblog?
+    status_for_destroy.discard
     RemovalWorker.perform_async(status_for_destroy.id)
 
     render json: @status, serializer: REST::StatusSerializer, relationships: StatusRelationshipsPresenter.new([@status], current_user&.account_id, reblogs_map: @reblogs_map)
@@ -30,7 +31,7 @@ class Api::V1::Statuses::ReblogsController < Api::BaseController
   end
 
   def status_for_destroy
-    current_user.account.statuses.where(reblog_of_id: params[:status_id]).first!
+    @status_for_destroy ||= current_user.account.statuses.where(reblog_of_id: params[:status_id]).first!
   end
 
   def reblog_params
diff --git a/app/controllers/api/v1/statuses_controller.rb b/app/controllers/api/v1/statuses_controller.rb
index 39ca56482..bba3c0651 100644
--- a/app/controllers/api/v1/statuses_controller.rb
+++ b/app/controllers/api/v1/statuses_controller.rb
@@ -53,6 +53,7 @@ class Api::V1::StatusesController < Api::BaseController
     @status = Status.where(account_id: current_user.account).find(params[:id])
     authorize @status, :destroy?
 
+    @status.discard
     RemovalWorker.perform_async(@status.id, redraft: true)
 
     render json: @status, serializer: REST::StatusSerializer, source_requested: true
diff --git a/app/models/form/status_batch.rb b/app/models/form/status_batch.rb
index 831d8b7c5..e09cc2594 100644
--- a/app/models/form/status_batch.rb
+++ b/app/models/form/status_batch.rb
@@ -34,6 +34,7 @@ class Form::StatusBatch
 
   def delete_statuses
     Status.where(id: status_ids).reorder(nil).find_each do |status|
+      status.discard
       RemovalWorker.perform_async(status.id, redraft: false)
       Tombstone.find_or_create_by(uri: status.uri, account: status.account, by_moderator: true)
       log_action :destroy, status
diff --git a/app/models/report.rb b/app/models/report.rb
index 5192ceef7..1e707ff1c 100644
--- a/app/models/report.rb
+++ b/app/models/report.rb
@@ -43,7 +43,7 @@ class Report < ApplicationRecord
   end
 
   def statuses
-    Status.where(id: status_ids).includes(:account, :media_attachments, :mentions)
+    Status.with_discarded.where(id: status_ids).includes(:account, :media_attachments, :mentions)
   end
 
   def media_attachments
diff --git a/app/models/status.rb b/app/models/status.rb
index 0538c4e9e..9cfaddcec 100644
--- a/app/models/status.rb
+++ b/app/models/status.rb
@@ -22,15 +22,19 @@
 #  application_id         :bigint(8)
 #  in_reply_to_account_id :bigint(8)
 #  poll_id                :bigint(8)
+#  deleted_at             :datetime
 #
 
 class Status < ApplicationRecord
   before_destroy :unlink_from_conversations
 
+  include Discard::Model
   include Paginable
   include Cacheable
   include StatusThreadingConcern
 
+  self.discard_column = :deleted_at
+
   # If `override_timestamps` is set at creation time, Snowflake ID creation
   # will be based on current time instead of `created_at`
   attr_accessor :override_timestamps
@@ -72,7 +76,7 @@ class Status < ApplicationRecord
 
   accepts_nested_attributes_for :poll
 
-  default_scope { recent }
+  default_scope { recent.kept }
 
   scope :recent, -> { reorder(id: :desc) }
   scope :remote, -> { where(local: false).where.not(uri: nil) }
diff --git a/app/views/admin/reports/_status.html.haml b/app/views/admin/reports/_status.html.haml
index 9376db7ff..6facc0a56 100644
--- a/app/views/admin/reports/_status.html.haml
+++ b/app/views/admin/reports/_status.html.haml
@@ -16,11 +16,14 @@
         - video = status.proper.media_attachments.first
         = react_component :video, src: video.file.url(:original), preview: video.file.url(:small), sensitive: !current_account&.user&.show_all_media? && status.proper.sensitive? || current_account&.user&.hide_all_media?, width: 610, height: 343, inline: true, alt: video.description
       - else
-        = react_component :media_gallery, height: 343, sensitive: !current_account&.user&.show_all_media? && status.sensitive? || current_account&.user&.hide_all_media?, 'autoPlayGif': current_account&.user&.setting_auto_play_gif, media: status.proper.media_attachments.map { |a| ActiveModelSerializers::SerializableResource.new(a, serializer: REST::MediaAttachmentSerializer).as_json }
+        = react_component :media_gallery, height: 343, sensitive: !current_account&.user&.show_all_media? && status.proper.sensitive? || current_account&.user&.hide_all_media?, 'autoPlayGif': current_account&.user&.setting_auto_play_gif, media: status.proper.media_attachments.map { |a| ActiveModelSerializers::SerializableResource.new(a, serializer: REST::MediaAttachmentSerializer).as_json }
 
     .detailed-status__meta
       = link_to ActivityPub::TagManager.instance.url_for(status), class: 'detailed-status__datetime', target: stream_link_target, rel: 'noopener' do
         %time.formatted{ datetime: status.created_at.iso8601, title: l(status.created_at) }= l(status.created_at)
+      - if status.discarded?
+        ·
+        %span.negative-hint= t('admin.statuses.deleted')
       ·
       - if status.reblog?
         = fa_icon('retweet fw')
diff --git a/app/workers/removal_worker.rb b/app/workers/removal_worker.rb
index 14423a4fb..2a1eaa89b 100644
--- a/app/workers/removal_worker.rb
+++ b/app/workers/removal_worker.rb
@@ -4,7 +4,7 @@ class RemovalWorker
   include Sidekiq::Worker
 
   def perform(status_id, options = {})
-    RemoveStatusService.new.call(Status.find(status_id), **options.symbolize_keys)
+    RemoveStatusService.new.call(Status.with_discarded.find(status_id), **options.symbolize_keys)
   rescue ActiveRecord::RecordNotFound
     true
   end
diff --git a/config/locales/en.yml b/config/locales/en.yml
index 8d267065c..a50dcb8a5 100644
--- a/config/locales/en.yml
+++ b/config/locales/en.yml
@@ -499,6 +499,7 @@ en:
         delete: Delete
         nsfw_off: Mark as not sensitive
         nsfw_on: Mark as sensitive
+      deleted: Deleted
       failed_to_execute: Failed to execute
       media:
         title: Media
diff --git a/db/migrate/20190819134503_add_deleted_at_to_statuses.rb b/db/migrate/20190819134503_add_deleted_at_to_statuses.rb
new file mode 100644
index 000000000..5af109097
--- /dev/null
+++ b/db/migrate/20190819134503_add_deleted_at_to_statuses.rb
@@ -0,0 +1,5 @@
+class AddDeletedAtToStatuses < ActiveRecord::Migration[5.2]
+  def change
+    add_column :statuses, :deleted_at, :datetime
+  end
+end
diff --git a/db/migrate/20190820003045_update_statuses_index.rb b/db/migrate/20190820003045_update_statuses_index.rb
new file mode 100644
index 000000000..5c2ea1f6a
--- /dev/null
+++ b/db/migrate/20190820003045_update_statuses_index.rb
@@ -0,0 +1,13 @@
+class UpdateStatusesIndex < ActiveRecord::Migration[5.2]
+  disable_ddl_transaction!
+
+  def up
+    safety_assured { add_index :statuses, [:account_id, :id, :visibility, :updated_at], where: 'deleted_at IS NULL', order: { id: :desc }, algorithm: :concurrently, name: :index_statuses_20190820 }
+    remove_index :statuses, name: :index_statuses_20180106
+  end
+
+  def down
+    safety_assured { add_index :statuses, [:account_id, :id, :visibility, :updated_at], order: { id: :desc }, algorithm: :concurrently, name: :index_statuses_20180106 }
+    remove_index :statuses, name: :index_statuses_20190820
+  end
+end
diff --git a/db/schema.rb b/db/schema.rb
index 18f615d61..afa6d724c 100644
--- a/db/schema.rb
+++ b/db/schema.rb
@@ -10,7 +10,7 @@
 #
 # It's strongly recommended that you check this file into your version control system.
 
-ActiveRecord::Schema.define(version: 2019_08_15_225426) do
+ActiveRecord::Schema.define(version: 2019_08_20_003045) do
 
   # These are extensions that must be enabled in order to support this database
   enable_extension "plpgsql"
@@ -644,7 +644,8 @@ ActiveRecord::Schema.define(version: 2019_08_15_225426) do
     t.bigint "application_id"
     t.bigint "in_reply_to_account_id"
     t.bigint "poll_id"
-    t.index ["account_id", "id", "visibility", "updated_at"], name: "index_statuses_20180106", order: { id: :desc }
+    t.datetime "deleted_at"
+    t.index ["account_id", "id", "visibility", "updated_at"], name: "index_statuses_20190820", order: { id: :desc }, where: "(deleted_at IS NULL)"
     t.index ["in_reply_to_account_id"], name: "index_statuses_on_in_reply_to_account_id"
     t.index ["in_reply_to_id"], name: "index_statuses_on_in_reply_to_id"
     t.index ["reblog_of_id", "account_id"], name: "index_statuses_on_reblog_of_id_and_account_id"
-- 
cgit 


From 73ca0bb925cb036f824262ab292a157a40a515d0 Mon Sep 17 00:00:00 2001
From: Eugen Rochko <eugen@zeonfederated.com>
Date: Fri, 23 Aug 2019 22:37:23 +0200
Subject: Add option to include reported statuses in warning e-mail (#11639)

---
 .../admin/account_actions_controller.rb            |  4 ++--
 app/javascript/styles/mailer.scss                  |  7 ++++++
 app/mailers/user_mailer.rb                         |  4 +++-
 app/models/admin/account_action.rb                 | 22 +++++++++++++-----
 app/views/admin/account_actions/new.html.haml      |  4 ++++
 app/views/notification_mailer/_status.html.haml    |  8 ++++++-
 app/views/user_mailer/warning.html.haml            | 27 +++++++++++++++++++++-
 app/views/user_mailer/warning.text.erb             | 13 +++++++++++
 config/locales/en.yml                              |  2 ++
 config/locales/simple_form.en.yml                  |  2 ++
 spec/mailers/previews/user_mailer_preview.rb       |  2 +-
 spec/models/admin/account_action_spec.rb           |  4 ++--
 12 files changed, 85 insertions(+), 14 deletions(-)

(limited to 'app/models')

diff --git a/app/controllers/admin/account_actions_controller.rb b/app/controllers/admin/account_actions_controller.rb
index a2cea461e..ea56fa0ac 100644
--- a/app/controllers/admin/account_actions_controller.rb
+++ b/app/controllers/admin/account_actions_controller.rb
@@ -5,7 +5,7 @@ module Admin
     before_action :set_account
 
     def new
-      @account_action  = Admin::AccountAction.new(type: params[:type], report_id: params[:report_id], send_email_notification: true)
+      @account_action  = Admin::AccountAction.new(type: params[:type], report_id: params[:report_id], send_email_notification: true, include_statuses: true)
       @warning_presets = AccountWarningPreset.all
     end
 
@@ -30,7 +30,7 @@ module Admin
     end
 
     def resource_params
-      params.require(:admin_account_action).permit(:type, :report_id, :warning_preset_id, :text, :send_email_notification)
+      params.require(:admin_account_action).permit(:type, :report_id, :warning_preset_id, :text, :send_email_notification, :include_statuses)
     end
   end
 end
diff --git a/app/javascript/styles/mailer.scss b/app/javascript/styles/mailer.scss
index b4fb1d709..e25a80c04 100644
--- a/app/javascript/styles/mailer.scss
+++ b/app/javascript/styles/mailer.scss
@@ -457,6 +457,13 @@ h5 {
 .status {
   padding-bottom: 32px;
 
+  &--highlighted {
+    border: 1px solid lighten($ui-base-color, 8%);
+    border-radius: 4px;
+    padding-bottom: 16px;
+    margin-bottom: 16px;
+  }
+
   .status-header {
     td {
       font-size: 14px;
diff --git a/app/mailers/user_mailer.rb b/app/mailers/user_mailer.rb
index 8f3a4ab3a..b41004acc 100644
--- a/app/mailers/user_mailer.rb
+++ b/app/mailers/user_mailer.rb
@@ -5,6 +5,7 @@ class UserMailer < Devise::Mailer
 
   helper :application
   helper :instance
+  helper :statuses
 
   add_template_helper RoutingHelper
 
@@ -79,10 +80,11 @@ class UserMailer < Devise::Mailer
     end
   end
 
-  def warning(user, warning)
+  def warning(user, warning, status_ids = nil)
     @resource = user
     @warning  = warning
     @instance = Rails.configuration.x.local_domain
+    @statuses = Status.where(id: status_ids).includes(:account) if status_ids.is_a?(Array)
 
     I18n.with_locale(@resource.locale || I18n.default_locale) do
       mail to: @resource.email,
diff --git a/app/models/admin/account_action.rb b/app/models/admin/account_action.rb
index bdbd342fb..c7da8b52c 100644
--- a/app/models/admin/account_action.rb
+++ b/app/models/admin/account_action.rb
@@ -19,20 +19,25 @@ class Admin::AccountAction
                 :report_id,
                 :warning_preset_id
 
-  attr_reader :warning, :send_email_notification
+  attr_reader :warning, :send_email_notification, :include_statuses
 
   def send_email_notification=(value)
     @send_email_notification = ActiveModel::Type::Boolean.new.cast(value)
   end
 
+  def include_statuses=(value)
+    @include_statuses = ActiveModel::Type::Boolean.new.cast(value)
+  end
+
   def save!
     ApplicationRecord.transaction do
       process_action!
       process_warning!
     end
 
-    queue_email!
+    process_email!
     process_reports!
+    process_queue!
   end
 
   def report
@@ -110,7 +115,6 @@ class Admin::AccountAction
     authorize(target_account, :suspend?)
     log_action(:suspend, target_account)
     target_account.suspend!
-    queue_suspension_worker!
   end
 
   def text_for_warning
@@ -121,16 +125,22 @@ class Admin::AccountAction
     Admin::SuspensionWorker.perform_async(target_account.id)
   end
 
-  def queue_email!
-    return unless warnable?
+  def process_queue!
+    queue_suspension_worker! if type == 'suspend'
+  end
 
-    UserMailer.warning(target_account.user, warning).deliver_later!
+  def process_email!
+    UserMailer.warning(target_account.user, warning, status_ids).deliver_now! if warnable?
   end
 
   def warnable?
     send_email_notification && target_account.local?
   end
 
+  def status_ids
+    @report.status_ids if @report && include_statuses
+  end
+
   def warning_preset
     @warning_preset ||= AccountWarningPreset.find(warning_preset_id) if warning_preset_id.present?
   end
diff --git a/app/views/admin/account_actions/new.html.haml b/app/views/admin/account_actions/new.html.haml
index 97286c8e5..20fbeef33 100644
--- a/app/views/admin/account_actions/new.html.haml
+++ b/app/views/admin/account_actions/new.html.haml
@@ -13,6 +13,10 @@
     .fields-group
       = f.input :send_email_notification, as: :boolean, wrapper: :with_label
 
+    - if params[:report_id].present?
+      .fields-group
+        = f.input :include_statuses, as: :boolean, wrapper: :with_label
+
     %hr.spacer/
 
     - unless @warning_presets.empty?
diff --git a/app/views/notification_mailer/_status.html.haml b/app/views/notification_mailer/_status.html.haml
index 57b5688bd..40f3aa88a 100644
--- a/app/views/notification_mailer/_status.html.haml
+++ b/app/views/notification_mailer/_status.html.haml
@@ -1,4 +1,5 @@
 - i ||= 0
+- highlighted ||= false
 
 %table.email-table{ cellspacing: 0, cellpadding: 0, dir: 'ltr' }
   %tbody
@@ -14,7 +15,7 @@
                       %table.column{ cellspacing: 0, cellpadding: 0 }
                         %tbody
                           %tr
-                            %td.column-cell.padded.status
+                            %td.column-cell.padded.status{ class: highlighted ? 'status--highlighted' : '' }
                               %table.status-header{ cellspacing: 0, cellpadding: 0 }
                                 %tbody
                                   %tr
@@ -32,5 +33,10 @@
                               %div{ dir: rtl_status?(status) ? 'rtl' : 'ltr' }
                                 = Formatter.instance.format(status)
 
+                                - if status.media_attachments.size > 0
+                                  %p
+                                    - status.media_attachments.each do |a|
+                                      = link_to medium_url(a), medium_url(a)
+
                               %p.status-footer
                                 = link_to l(status.created_at), web_url("statuses/#{status.id}")
diff --git a/app/views/user_mailer/warning.html.haml b/app/views/user_mailer/warning.html.haml
index 72ea5e5d2..030a57bb4 100644
--- a/app/views/user_mailer/warning.html.haml
+++ b/app/views/user_mailer/warning.html.haml
@@ -42,6 +42,14 @@
                               - unless @warning.text.blank?
                                 = Formatter.instance.linkify(@warning.text)
 
+                              - unless @statuses.empty?
+                                %p
+                                  %strong= t('user_mailer.warning.statuses')
+
+- unless @statuses.empty?
+  - @statuses.each_with_index do |status, i|
+    = render 'notification_mailer/status', status: status, i: i + 1, highlighted: true
+
 %table.email-table{ cellspacing: 0, cellpadding: 0 }
   %tbody
     %tr
@@ -50,7 +58,7 @@
           %table.content-section{ cellspacing: 0, cellpadding: 0 }
             %tbody
               %tr
-                %td.content-cell
+                %td.content-cell{ class: @statuses.empty? ? '' : 'content-start' }
                   %table.column{ cellspacing: 0, cellpadding: 0 }
                     %tbody
                       %tr
@@ -61,3 +69,20 @@
                                 %td.button-primary
                                   = link_to about_more_url do
                                     %span= t 'user_mailer.warning.review_server_policies'
+
+%table.email-table{ cellspacing: 0, cellpadding: 0 }
+  %tbody
+    %tr
+      %td.email-body
+        .email-container
+          %table.content-section{ cellspacing: 0, cellpadding: 0 }
+            %tbody
+              %tr
+                %td.content-cell
+                  .email-row
+                    .col-6
+                      %table.column{ cellspacing: 0, cellpadding: 0 }
+                        %tbody
+                          %tr
+                            %td.column-cell.text-center
+                              %p= t 'user_mailer.warning.get_in_touch', instance: @instance
diff --git a/app/views/user_mailer/warning.text.erb b/app/views/user_mailer/warning.text.erb
index b4f2402cb..24c1f86f2 100644
--- a/app/views/user_mailer/warning.text.erb
+++ b/app/views/user_mailer/warning.text.erb
@@ -7,3 +7,16 @@
 
 <% end %>
 <%= @warning.text %>
+<% unless @statuses.empty? %>
+<%= t('user_mailer.warning.statuses') %>
+
+<% @statuses.each do |status| %>
+
+<%= render 'notification_mailer/status', status: status %>
+---
+<% end %>
+<% else %>
+---
+<% end %>
+
+<%= t 'user_mailer.warning.get_in_touch', instance: @instance %>
diff --git a/config/locales/en.yml b/config/locales/en.yml
index a50dcb8a5..ee78e4720 100644
--- a/config/locales/en.yml
+++ b/config/locales/en.yml
@@ -1115,7 +1115,9 @@ en:
         disable: While your account is frozen, your account data remains intact, but you cannot perform any actions until it is unlocked.
         silence: While your account is limited, only people who are already following you will see your toots on this server, and you may be excluded from various public listings. However, others may still manually follow you.
         suspend: Your account has been suspended, and all of your toots and your uploaded media files have been irreversibly removed from this server, and servers where you had followers.
+      get_in_touch: You can reply to this e-mail to get in touch with the staff of %{instance}.
       review_server_policies: Review server policies
+      statuses: 'Specifically, for:'
       subject:
         disable: Your account %{acct} has been frozen
         none: Warning for %{acct}
diff --git a/config/locales/simple_form.en.yml b/config/locales/simple_form.en.yml
index 98f0843d0..cfaa6e666 100644
--- a/config/locales/simple_form.en.yml
+++ b/config/locales/simple_form.en.yml
@@ -5,6 +5,7 @@ en:
       account_warning_preset:
         text: You can use toot syntax, such as URLs, hashtags and mentions
       admin_account_action:
+        include_statuses: The user will see which toots have caused the moderation action or warning
         send_email_notification: The user will receive an explanation of what happened with their account
         text_html: Optional. You can use toot syntax. You can <a href="%{path}">add warning presets</a> to save time
         type_html: Choose what to do with <strong>%{acct}</strong>
@@ -60,6 +61,7 @@ en:
       account_warning_preset:
         text: Preset text
       admin_account_action:
+        include_statuses: Include reported toots in the e-mail
         send_email_notification: Notify the user per e-mail
         text: Custom warning
         type: Action
diff --git a/spec/mailers/previews/user_mailer_preview.rb b/spec/mailers/previews/user_mailer_preview.rb
index 53c836494..ead3b3baa 100644
--- a/spec/mailers/previews/user_mailer_preview.rb
+++ b/spec/mailers/previews/user_mailer_preview.rb
@@ -42,6 +42,6 @@ class UserMailerPreview < ActionMailer::Preview
 
   # Preview this email at http://localhost:3000/rails/mailers/user_mailer/warning
   def warning
-    UserMailer.warning(User.first, AccountWarning.new(text: '', action: :silence))
+    UserMailer.warning(User.first, AccountWarning.new(text: '', action: :silence), [Status.first.id])
   end
 end
diff --git a/spec/models/admin/account_action_spec.rb b/spec/models/admin/account_action_spec.rb
index a3db60cfc..87fc28500 100644
--- a/spec/models/admin/account_action_spec.rb
+++ b/spec/models/admin/account_action_spec.rb
@@ -58,8 +58,8 @@ RSpec.describe Admin::AccountAction, type: :model do
       end.to change { Admin::ActionLog.count }.by 1
     end
 
-    it 'calls queue_email!' do
-      expect(account_action).to receive(:queue_email!)
+    it 'calls process_email!' do
+      expect(account_action).to receive(:process_email!)
       subject
     end
 
-- 
cgit 


From cb447b28c403c7db32e3e3d7c2510004287edfda Mon Sep 17 00:00:00 2001
From: Eugen Rochko <eugen@zeonfederated.com>
Date: Fri, 30 Aug 2019 00:14:36 +0200
Subject: Add profile directory to web UI (#11688)

* Add profile directory to web UI

* Add a line of bio to the directory
---
 app/controllers/api/v1/directories_controller.rb   |  30 +++
 app/controllers/directories_controller.rb          |   8 +-
 app/javascript/mastodon/actions/directory.js       |  61 +++++
 app/javascript/mastodon/components/radio_button.js |  35 +++
 .../features/directory/components/account_card.js  | 149 +++++++++++
 .../mastodon/features/directory/index.js           | 171 +++++++++++++
 .../mastodon/features/getting_started/index.js     |   4 +-
 .../containers/column_settings_container.js        |   2 +-
 .../features/ui/components/columns_area.js         |  14 +-
 .../features/ui/components/navigation_panel.js     |   2 +-
 app/javascript/mastodon/features/ui/index.js       |   2 +
 .../mastodon/features/ui/util/async-components.js  |   4 +
 app/javascript/mastodon/reducers/user_lists.js     |  18 ++
 app/javascript/styles/mastodon/components.scss     | 284 +++++++++++++++------
 app/lib/activitypub/adapter.rb                     |   1 +
 app/models/account.rb                              |   7 +-
 app/serializers/activitypub/actor_serializer.rb    |   6 +-
 app/serializers/rest/account_serializer.rb         |   2 +-
 .../activitypub/process_account_service.rb         |   1 +
 app/views/application/_card.html.haml              |   2 +-
 app/views/directories/index.html.haml              |  60 +----
 app/views/settings/profiles/show.html.haml         |   2 +-
 config/locales/en.yml                              |   6 -
 config/locales/simple_form.en.yml                  |   2 +-
 config/routes.rb                                   |   1 +
 25 files changed, 715 insertions(+), 159 deletions(-)
 create mode 100644 app/controllers/api/v1/directories_controller.rb
 create mode 100644 app/javascript/mastodon/actions/directory.js
 create mode 100644 app/javascript/mastodon/components/radio_button.js
 create mode 100644 app/javascript/mastodon/features/directory/components/account_card.js
 create mode 100644 app/javascript/mastodon/features/directory/index.js

(limited to 'app/models')

diff --git a/app/controllers/api/v1/directories_controller.rb b/app/controllers/api/v1/directories_controller.rb
new file mode 100644
index 000000000..c91543e3a
--- /dev/null
+++ b/app/controllers/api/v1/directories_controller.rb
@@ -0,0 +1,30 @@
+# frozen_string_literal: true
+
+class Api::V1::DirectoriesController < Api::BaseController
+  before_action :require_enabled!
+  before_action :set_accounts
+
+  def show
+    render json: @accounts, each_serializer: REST::AccountSerializer
+  end
+
+  private
+
+  def require_enabled!
+    return not_found unless Setting.profile_directory
+  end
+
+  def set_accounts
+    @accounts = accounts_scope.offset(params[:offset]).limit(limit_param(DEFAULT_ACCOUNTS_LIMIT))
+  end
+
+  def accounts_scope
+    Account.discoverable.tap do |scope|
+      scope.merge!(Account.local)                                          if truthy_param?(:local)
+      scope.merge!(Account.by_recent_status)                               if params[:order].blank? || params[:order] == 'active'
+      scope.merge!(Account.order(id: :desc))                               if params[:order] == 'new'
+      scope.merge!(Account.not_excluded_by_account(current_account))       if current_account
+      scope.merge!(Account.not_domain_blocked_by_account(current_account)) if current_account && !truthy_param?(:local)
+    end
+  end
+end
diff --git a/app/controllers/directories_controller.rb b/app/controllers/directories_controller.rb
index a5c47b515..7244f02f0 100644
--- a/app/controllers/directories_controller.rb
+++ b/app/controllers/directories_controller.rb
@@ -7,7 +7,6 @@ class DirectoriesController < ApplicationController
   before_action :require_enabled!
   before_action :set_instance_presenter
   before_action :set_tag, only: :show
-  before_action :set_tags
   before_action :set_accounts
 
   def index
@@ -28,13 +27,10 @@ class DirectoriesController < ApplicationController
     @tag = Tag.discoverable.find_normalized!(params[:id])
   end
 
-  def set_tags
-    @tags = Tag.discoverable.limit(30).reject { |tag| tag.cached_sample_accounts.empty? }
-  end
-
   def set_accounts
-    @accounts = Account.discoverable.by_recent_status.page(params[:page]).per(40).tap do |query|
+    @accounts = Account.local.discoverable.by_recent_status.page(params[:page]).per(15).tap do |query|
       query.merge!(Account.tagged_with(@tag.id)) if @tag
+      query.merge!(Account.not_excluded_by_account(current_account)) if current_account
     end
   end
 
diff --git a/app/javascript/mastodon/actions/directory.js b/app/javascript/mastodon/actions/directory.js
new file mode 100644
index 000000000..4b2b6dd56
--- /dev/null
+++ b/app/javascript/mastodon/actions/directory.js
@@ -0,0 +1,61 @@
+import api from '../api';
+import { importFetchedAccounts } from './importer';
+import { fetchRelationships } from './accounts';
+
+export const DIRECTORY_FETCH_REQUEST = 'DIRECTORY_FETCH_REQUEST';
+export const DIRECTORY_FETCH_SUCCESS = 'DIRECTORY_FETCH_SUCCESS';
+export const DIRECTORY_FETCH_FAIL    = 'DIRECTORY_FETCH_FAIL';
+
+export const DIRECTORY_EXPAND_REQUEST = 'DIRECTORY_EXPAND_REQUEST';
+export const DIRECTORY_EXPAND_SUCCESS = 'DIRECTORY_EXPAND_SUCCESS';
+export const DIRECTORY_EXPAND_FAIL    = 'DIRECTORY_EXPAND_FAIL';
+
+export const fetchDirectory = params => (dispatch, getState) => {
+  dispatch(fetchDirectoryRequest());
+
+  api(getState).get('/api/v1/directory', { params: { ...params, limit: 20 } }).then(({ data }) => {
+    dispatch(importFetchedAccounts(data));
+    dispatch(fetchDirectorySuccess(data));
+    dispatch(fetchRelationships(data.map(x => x.id)));
+  }).catch(error => dispatch(fetchDirectoryFail(error)));
+};
+
+export const fetchDirectoryRequest = () => ({
+  type: DIRECTORY_FETCH_REQUEST,
+});
+
+export const fetchDirectorySuccess = accounts => ({
+  type: DIRECTORY_FETCH_SUCCESS,
+  accounts,
+});
+
+export const fetchDirectoryFail = error => ({
+  type: DIRECTORY_FETCH_FAIL,
+  error,
+});
+
+export const expandDirectory = params => (dispatch, getState) => {
+  dispatch(expandDirectoryRequest());
+
+  const loadedItems = getState().getIn(['user_lists', 'directory', 'items']).size;
+
+  api(getState).get('/api/v1/directory', { params: { ...params, offset: loadedItems, limit: 20 } }).then(({ data }) => {
+    dispatch(importFetchedAccounts(data));
+    dispatch(expandDirectorySuccess(data));
+    dispatch(fetchRelationships(data.map(x => x.id)));
+  }).catch(error => dispatch(expandDirectoryFail(error)));
+};
+
+export const expandDirectoryRequest = () => ({
+  type: DIRECTORY_EXPAND_REQUEST,
+});
+
+export const expandDirectorySuccess = accounts => ({
+  type: DIRECTORY_EXPAND_SUCCESS,
+  accounts,
+});
+
+export const expandDirectoryFail = error => ({
+  type: DIRECTORY_EXPAND_FAIL,
+  error,
+});
diff --git a/app/javascript/mastodon/components/radio_button.js b/app/javascript/mastodon/components/radio_button.js
new file mode 100644
index 000000000..0496fa286
--- /dev/null
+++ b/app/javascript/mastodon/components/radio_button.js
@@ -0,0 +1,35 @@
+import React from 'react';
+import PropTypes from 'prop-types';
+import classNames from 'classnames';
+
+export default class RadioButton extends React.PureComponent {
+
+  static propTypes = {
+    value: PropTypes.string.isRequired,
+    checked: PropTypes.bool,
+    name: PropTypes.string.isRequired,
+    onChange: PropTypes.func.isRequired,
+    label: PropTypes.node.isRequired,
+  };
+
+  render () {
+    const { name, value, checked, onChange, label } = this.props;
+
+    return (
+      <label className='radio-button'>
+        <input
+          name={name}
+          type='radio'
+          value={value}
+          checked={checked}
+          onChange={onChange}
+        />
+
+        <span className={classNames('radio-button__input', { checked })} />
+
+        <span>{label}</span>
+      </label>
+    );
+  }
+
+}
diff --git a/app/javascript/mastodon/features/directory/components/account_card.js b/app/javascript/mastodon/features/directory/components/account_card.js
new file mode 100644
index 000000000..a9c9976be
--- /dev/null
+++ b/app/javascript/mastodon/features/directory/components/account_card.js
@@ -0,0 +1,149 @@
+import React from 'react';
+import ImmutablePureComponent from 'react-immutable-pure-component';
+import ImmutablePropTypes from 'react-immutable-proptypes';
+import PropTypes from 'prop-types';
+import { connect } from 'react-redux';
+import { makeGetAccount } from 'mastodon/selectors';
+import Avatar from 'mastodon/components/avatar';
+import DisplayName from 'mastodon/components/display_name';
+import Permalink from 'mastodon/components/permalink';
+import RelativeTimestamp from 'mastodon/components/relative_timestamp';
+import IconButton from 'mastodon/components/icon_button';
+import { FormattedMessage, injectIntl, defineMessages } from 'react-intl';
+import { autoPlayGif, me, unfollowModal } from 'mastodon/initial_state';
+import { shortNumberFormat } from 'mastodon/utils/numbers';
+import { followAccount, unfollowAccount, blockAccount, unblockAccount, unmuteAccount } from 'mastodon/actions/accounts';
+import { openModal } from 'mastodon/actions/modal';
+import { initMuteModal } from 'mastodon/actions/mutes';
+
+const messages = defineMessages({
+  follow: { id: 'account.follow', defaultMessage: 'Follow' },
+  unfollow: { id: 'account.unfollow', defaultMessage: 'Unfollow' },
+  requested: { id: 'account.requested', defaultMessage: 'Awaiting approval' },
+  unblock: { id: 'account.unblock', defaultMessage: 'Unblock @{name}' },
+  unmute: { id: 'account.unmute', defaultMessage: 'Unmute @{name}' },
+});
+
+const makeMapStateToProps = () => {
+  const getAccount = makeGetAccount();
+
+  const mapStateToProps = (state, { id }) => ({
+    account: getAccount(state, id),
+  });
+
+  return mapStateToProps;
+};
+
+const mapDispatchToProps = (dispatch, { intl }) => ({
+
+  onFollow (account) {
+    if (account.getIn(['relationship', 'following']) || account.getIn(['relationship', 'requested'])) {
+      if (unfollowModal) {
+        dispatch(openModal('CONFIRM', {
+          message: <FormattedMessage id='confirmations.unfollow.message' defaultMessage='Are you sure you want to unfollow {name}?' values={{ name: <strong>@{account.get('acct')}</strong> }} />,
+          confirm: intl.formatMessage(messages.unfollowConfirm),
+          onConfirm: () => dispatch(unfollowAccount(account.get('id'))),
+        }));
+      } else {
+        dispatch(unfollowAccount(account.get('id')));
+      }
+    } else {
+      dispatch(followAccount(account.get('id')));
+    }
+  },
+
+  onBlock (account) {
+    if (account.getIn(['relationship', 'blocking'])) {
+      dispatch(unblockAccount(account.get('id')));
+    } else {
+      dispatch(blockAccount(account.get('id')));
+    }
+  },
+
+  onMute (account) {
+    if (account.getIn(['relationship', 'muting'])) {
+      dispatch(unmuteAccount(account.get('id')));
+    } else {
+      dispatch(initMuteModal(account));
+    }
+  },
+
+});
+
+export default @injectIntl
+@connect(makeMapStateToProps, mapDispatchToProps)
+class AccountCard extends ImmutablePureComponent {
+
+  static propTypes = {
+    account: ImmutablePropTypes.map.isRequired,
+    intl: PropTypes.object.isRequired,
+    onFollow: PropTypes.func.isRequired,
+    onBlock: PropTypes.func.isRequired,
+    onMute: PropTypes.func.isRequired,
+  };
+
+  handleFollow = () => {
+    this.props.onFollow(this.props.account);
+  }
+
+  handleBlock = () => {
+    this.props.onBlock(this.props.account);
+  }
+
+  handleMute = () => {
+    this.props.onMute(this.props.account);
+  }
+
+  render () {
+    const { account, intl } = this.props;
+
+    let buttons;
+
+    if (account.get('id') !== me && account.get('relationship', null) !== null) {
+      const following = account.getIn(['relationship', 'following']);
+      const requested = account.getIn(['relationship', 'requested']);
+      const blocking  = account.getIn(['relationship', 'blocking']);
+      const muting    = account.getIn(['relationship', 'muting']);
+
+      if (requested) {
+        buttons = <IconButton disabled icon='hourglass' title={intl.formatMessage(messages.requested)} />;
+      } else if (blocking) {
+        buttons = <IconButton active icon='unlock' title={intl.formatMessage(messages.unblock, { name: account.get('username') })} onClick={this.handleBlock} />;
+      } else if (muting) {
+        buttons = <IconButton active icon='volume-up' title={intl.formatMessage(messages.unmute, { name: account.get('username') })} onClick={this.handleMute} />;
+      } else if (!account.get('moved') || following) {
+        buttons = <IconButton icon={following ? 'user-times' : 'user-plus'} title={intl.formatMessage(following ? messages.unfollow : messages.follow)} onClick={this.handleFollow} active={following} />;
+      }
+    }
+
+    return (
+      <div className='directory__card'>
+        <div className='directory__card__img'>
+          <img src={autoPlayGif ? account.get('header') : account.get('header_static')} alt='' className='parallax' />
+        </div>
+
+        <div className='directory__card__bar'>
+          <Permalink className='directory__card__bar__name' href={account.get('url')} to={`/accounts/${account.get('id')}`}>
+            <Avatar account={account} size={48} />
+            <DisplayName account={account} />
+          </Permalink>
+
+          <div className='directory__card__bar__relationship account__relationship'>
+            {buttons}
+          </div>
+        </div>
+
+        <div className='directory__card__extra'>
+          {account.get('note').length > 0 && account.get('note') !== '<p></p>' && <div className='account__header__content' dangerouslySetInnerHTML={{ __html: account.get('note_emojified') }} />}
+        </div>
+
+        <div className='directory__card__extra'>
+          <div className='accounts-table__count'>{shortNumberFormat(account.get('statuses_count'))} <small><FormattedMessage id='account.posts' defaultMessage='Toots' /></small></div>
+          <div className='accounts-table__count'>{shortNumberFormat(account.get('followers_count'))} <small><FormattedMessage id='account.followers' defaultMessage='Followers' /></small></div>
+          <div className='accounts-table__count'>{account.get('last_status_at') === null ? <FormattedMessage id='account.never_active' defaultMessage='Never' /> : <RelativeTimestamp timestamp={account.get('last_status_at')} />} <small><FormattedMessage id='account.last_status' defaultMessage='Last active' /></small></div>
+        </div>
+      </div>
+    );
+  }
+
+}
diff --git a/app/javascript/mastodon/features/directory/index.js b/app/javascript/mastodon/features/directory/index.js
new file mode 100644
index 000000000..2f91e759b
--- /dev/null
+++ b/app/javascript/mastodon/features/directory/index.js
@@ -0,0 +1,171 @@
+import React from 'react';
+import { connect } from 'react-redux';
+import { defineMessages, injectIntl } from 'react-intl';
+import PropTypes from 'prop-types';
+import ImmutablePropTypes from 'react-immutable-proptypes';
+import Column from 'mastodon/components/column';
+import ColumnHeader from 'mastodon/components/column_header';
+import { addColumn, removeColumn, moveColumn, changeColumnParams } from 'mastodon/actions/columns';
+import { fetchDirectory, expandDirectory } from 'mastodon/actions/directory';
+import { List as ImmutableList } from 'immutable';
+import AccountCard from './components/account_card';
+import RadioButton from 'mastodon/components/radio_button';
+import classNames from 'classnames';
+import LoadMore from 'mastodon/components/load_more';
+import { ScrollContainer } from 'react-router-scroll-4';
+
+const messages = defineMessages({
+  title: { id: 'column.directory', defaultMessage: 'Browse profiles' },
+  recentlyActive: { id: 'directory.recently_active', defaultMessage: 'Recently active' },
+  newArrivals: { id: 'directory.new_arrivals', defaultMessage: 'New arrivals' },
+  local: { id: 'directory.local', defaultMessage: 'From {domain} only' },
+  federated: { id: 'directory.federated', defaultMessage: 'From known fediverse' },
+});
+
+const mapStateToProps = state => ({
+  accountIds: state.getIn(['user_lists', 'directory', 'items'], ImmutableList()),
+  isLoading: state.getIn(['user_lists', 'directory', 'isLoading'], true),
+  domain: state.getIn(['meta', 'domain']),
+});
+
+export default @connect(mapStateToProps)
+@injectIntl
+class Directory extends React.PureComponent {
+
+  static contextTypes = {
+    router: PropTypes.object,
+  };
+
+  static propTypes = {
+    isLoading: PropTypes.bool,
+    accountIds: ImmutablePropTypes.list.isRequired,
+    dispatch: PropTypes.func.isRequired,
+    shouldUpdateScroll: PropTypes.func,
+    columnId: PropTypes.string,
+    intl: PropTypes.object.isRequired,
+    multiColumn: PropTypes.bool,
+    domain: PropTypes.string.isRequired,
+    params: PropTypes.shape({
+      order: PropTypes.string,
+      local: PropTypes.bool,
+    }),
+  };
+
+  state = {
+    order: null,
+    local: null,
+  };
+
+  handlePin = () => {
+    const { columnId, dispatch } = this.props;
+
+    if (columnId) {
+      dispatch(removeColumn(columnId));
+    } else {
+      dispatch(addColumn('DIRECTORY', this.getParams(this.props, this.state)));
+    }
+  }
+
+  getParams = (props, state) => ({
+    order: state.order === null ? (props.params.order || 'active') : state.order,
+    local: state.local === null ? (props.params.local || false) : state.local,
+  });
+
+  handleMove = dir => {
+    const { columnId, dispatch } = this.props;
+    dispatch(moveColumn(columnId, dir));
+  }
+
+  handleHeaderClick = () => {
+    this.column.scrollTop();
+  }
+
+  componentDidMount () {
+    const { dispatch } = this.props;
+    dispatch(fetchDirectory(this.getParams(this.props, this.state)));
+  }
+
+  componentDidUpdate (prevProps, prevState) {
+    const { dispatch } = this.props;
+    const paramsOld = this.getParams(prevProps, prevState);
+    const paramsNew = this.getParams(this.props, this.state);
+
+    if (paramsOld.order !== paramsNew.order || paramsOld.local !== paramsNew.local) {
+      dispatch(fetchDirectory(paramsNew));
+    }
+  }
+
+  setRef = c => {
+    this.column = c;
+  }
+
+  handleChangeOrder = e => {
+    const { dispatch, columnId } = this.props;
+
+    if (columnId) {
+      dispatch(changeColumnParams(columnId, ['order'], e.target.value));
+    } else {
+      this.setState({ order: e.target.value });
+    }
+  }
+
+  handleChangeLocal = e => {
+    const { dispatch, columnId } = this.props;
+
+    if (columnId) {
+      dispatch(changeColumnParams(columnId, ['local'], e.target.value === '1'));
+    } else {
+      this.setState({ local: e.target.value === '1' });
+    }
+  }
+
+  handleLoadMore = () => {
+    const { dispatch } = this.props;
+    dispatch(expandDirectory(this.getParams(this.props, this.state)));
+  }
+
+  render () {
+    const { isLoading, accountIds, intl, columnId, multiColumn, domain, shouldUpdateScroll } = this.props;
+    const { order, local }  = this.getParams(this.props, this.state);
+    const pinned = !!columnId;
+
+    const scrollableArea = (
+      <div className='scrollable' style={{ background: 'transparent' }}>
+        <div className='filter-form'>
+          <div className='filter-form__column' role='group'>
+            <RadioButton name='order' value='active' label={intl.formatMessage(messages.recentlyActive)} checked={order === 'active'} onChange={this.handleChangeOrder} />
+            <RadioButton name='order' value='new' label={intl.formatMessage(messages.newArrivals)} checked={order === 'new'} onChange={this.handleChangeOrder} />
+          </div>
+
+          <div className='filter-form__column' role='group'>
+            <RadioButton name='local' value='1' label={intl.formatMessage(messages.local, { domain })} checked={local} onChange={this.handleChangeLocal} />
+            <RadioButton name='local' value='0' label={intl.formatMessage(messages.federated)} checked={!local} onChange={this.handleChangeLocal} />
+          </div>
+        </div>
+
+        <div className={classNames('directory__list', { loading: isLoading })}>
+          {accountIds.map(accountId => <AccountCard id={accountId} key={accountId} />)}
+        </div>
+
+        <LoadMore onClick={this.handleLoadMore} visible={!isLoading} />
+      </div>
+    );
+
+    return (
+      <Column bindToDocument={!multiColumn} ref={this.setRef} label={intl.formatMessage(messages.title)}>
+        <ColumnHeader
+          icon='address-book-o'
+          title={intl.formatMessage(messages.title)}
+          onPin={this.handlePin}
+          onMove={this.handleMove}
+          onClick={this.handleHeaderClick}
+          pinned={pinned}
+          multiColumn={multiColumn}
+        />
+
+        {multiColumn && !pinned ? <ScrollContainer scrollKey='directory' shouldUpdateScroll={shouldUpdateScroll}>{scrollableArea}</ScrollContainer> : scrollableArea}
+      </Column>
+    );
+  }
+
+}
diff --git a/app/javascript/mastodon/features/getting_started/index.js b/app/javascript/mastodon/features/getting_started/index.js
index 6a122a750..f6d90580b 100644
--- a/app/javascript/mastodon/features/getting_started/index.js
+++ b/app/javascript/mastodon/features/getting_started/index.js
@@ -107,7 +107,7 @@ class GettingStarted extends ImmutablePureComponent {
 
       if (profile_directory) {
         navItems.push(
-          <ColumnLink key={i++} icon='address-book' text={intl.formatMessage(messages.profile_directory)} href='/explore' />
+          <ColumnLink key={i++} icon='address-book' text={intl.formatMessage(messages.profile_directory)} to='/directory' />
         );
 
         height += 48;
@@ -120,7 +120,7 @@ class GettingStarted extends ImmutablePureComponent {
       height += 34;
     } else if (profile_directory) {
       navItems.push(
-        <ColumnLink key={i++} icon='address-book' text={intl.formatMessage(messages.profile_directory)} href='/explore' />
+        <ColumnLink key={i++} icon='address-book' text={intl.formatMessage(messages.profile_directory)} to='/directory' />
       );
 
       height += 48;
diff --git a/app/javascript/mastodon/features/hashtag_timeline/containers/column_settings_container.js b/app/javascript/mastodon/features/hashtag_timeline/containers/column_settings_container.js
index c5098052c..5914bbeaf 100644
--- a/app/javascript/mastodon/features/hashtag_timeline/containers/column_settings_container.js
+++ b/app/javascript/mastodon/features/hashtag_timeline/containers/column_settings_container.js
@@ -20,7 +20,7 @@ const mapDispatchToProps = (dispatch, { columnId }) => ({
   },
 
   onLoad (value) {
-    return api().get('/api/v2/search', { params: { q: value } }).then(response => {
+    return api().get('/api/v2/search', { params: { q: value, type: 'hashtags' } }).then(response => {
       return (response.data.hashtags || []).map((tag) => {
         return { value: tag.name, label: `#${tag.name}` };
       });
diff --git a/app/javascript/mastodon/features/ui/components/columns_area.js b/app/javascript/mastodon/features/ui/components/columns_area.js
index 042e44e43..8a4e89b3d 100644
--- a/app/javascript/mastodon/features/ui/components/columns_area.js
+++ b/app/javascript/mastodon/features/ui/components/columns_area.js
@@ -12,7 +12,18 @@ import BundleContainer from '../containers/bundle_container';
 import ColumnLoading from './column_loading';
 import DrawerLoading from './drawer_loading';
 import BundleColumnError from './bundle_column_error';
-import { Compose, Notifications, HomeTimeline, CommunityTimeline, PublicTimeline, HashtagTimeline, DirectTimeline, FavouritedStatuses, ListTimeline } from '../../ui/util/async-components';
+import {
+  Compose,
+  Notifications,
+  HomeTimeline,
+  CommunityTimeline,
+  PublicTimeline,
+  HashtagTimeline,
+  DirectTimeline,
+  FavouritedStatuses,
+  ListTimeline,
+  Directory,
+} from '../../ui/util/async-components';
 import Icon from 'mastodon/components/icon';
 import ComposePanel from './compose_panel';
 import NavigationPanel from './navigation_panel';
@@ -30,6 +41,7 @@ const componentMap = {
   'DIRECT': DirectTimeline,
   'FAVOURITES': FavouritedStatuses,
   'LIST': ListTimeline,
+  'DIRECTORY': Directory,
 };
 
 const messages = defineMessages({
diff --git a/app/javascript/mastodon/features/ui/components/navigation_panel.js b/app/javascript/mastodon/features/ui/components/navigation_panel.js
index 64a40a9da..6f07778f2 100644
--- a/app/javascript/mastodon/features/ui/components/navigation_panel.js
+++ b/app/javascript/mastodon/features/ui/components/navigation_panel.js
@@ -18,6 +18,7 @@ const NavigationPanel = () => (
     <NavLink className='column-link column-link--transparent' to='/timelines/direct'><Icon className='column-link__icon' id='envelope' fixedWidth /><FormattedMessage id='navigation_bar.direct' defaultMessage='Direct messages' /></NavLink>
     <NavLink className='column-link column-link--transparent' to='/favourites'><Icon className='column-link__icon' id='star' fixedWidth /><FormattedMessage id='navigation_bar.favourites' defaultMessage='Favourites' /></NavLink>
     <NavLink className='column-link column-link--transparent' to='/lists'><Icon className='column-link__icon' id='list-ul' fixedWidth /><FormattedMessage id='navigation_bar.lists' defaultMessage='Lists' /></NavLink>
+    {profile_directory && <NavLink className='column-link column-link--transparent' to='/directory'><Icon className='column-link__icon' id='address-book-o' fixedWidth /><FormattedMessage id='getting_started.profile_directory' defaultMessage='Profile directory' /></NavLink>}
 
     <ListPanel />
 
@@ -25,7 +26,6 @@ const NavigationPanel = () => (
 
     <a className='column-link column-link--transparent' href='/settings/preferences'><Icon className='column-link__icon' id='cog' fixedWidth /><FormattedMessage id='navigation_bar.preferences' defaultMessage='Preferences' /></a>
     <a className='column-link column-link--transparent' href='/relationships'><Icon className='column-link__icon' id='users' fixedWidth /><FormattedMessage id='navigation_bar.follows_and_followers' defaultMessage='Follows and followers' /></a>
-    {!!profile_directory && <a className='column-link column-link--transparent' href='/explore'><Icon className='column-link__icon' id='address-book-o' fixedWidth /><FormattedMessage id='navigation_bar.profile_directory' defaultMessage='Profile directory' /></a>}
 
     {showTrends && <div className='flex-spacer' />}
     {showTrends && <TrendsContainer />}
diff --git a/app/javascript/mastodon/features/ui/index.js b/app/javascript/mastodon/features/ui/index.js
index 9d284c221..49c5c8d0e 100644
--- a/app/javascript/mastodon/features/ui/index.js
+++ b/app/javascript/mastodon/features/ui/index.js
@@ -47,6 +47,7 @@ import {
   PinnedStatuses,
   Lists,
   Search,
+  Directory,
 } from './util/async-components';
 import { me, forceSingleColumn } from '../../initial_state';
 import { previewState as previewMediaState } from './components/media_modal';
@@ -188,6 +189,7 @@ class SwitchingColumnsArea extends React.PureComponent {
           <WrappedRoute path='/pinned' component={PinnedStatuses} content={children} componentParams={{ shouldUpdateScroll: this.shouldUpdateScroll }} />
 
           <WrappedRoute path='/search' component={Search} content={children} />
+          <WrappedRoute path='/directory' component={Directory} content={children} componentParams={{ shouldUpdateScroll: this.shouldUpdateScroll }} />
 
           <WrappedRoute path='/statuses/new' component={Compose} content={children} />
           <WrappedRoute path='/statuses/:statusId' exact component={Status} content={children} componentParams={{ shouldUpdateScroll: this.shouldUpdateScroll }} />
diff --git a/app/javascript/mastodon/features/ui/util/async-components.js b/app/javascript/mastodon/features/ui/util/async-components.js
index a9b95c7b8..0084c1510 100644
--- a/app/javascript/mastodon/features/ui/util/async-components.js
+++ b/app/javascript/mastodon/features/ui/util/async-components.js
@@ -141,3 +141,7 @@ export function Tesseract () {
 export function Audio () {
   return import(/* webpackChunkName: "features/audio" */'../../audio');
 }
+
+export function Directory () {
+  return import(/* webpackChunkName: "features/directory" */'../../directory');
+}
diff --git a/app/javascript/mastodon/reducers/user_lists.js b/app/javascript/mastodon/reducers/user_lists.js
index 8db18c5dc..08e94022f 100644
--- a/app/javascript/mastodon/reducers/user_lists.js
+++ b/app/javascript/mastodon/reducers/user_lists.js
@@ -20,6 +20,14 @@ import {
   MUTES_FETCH_SUCCESS,
   MUTES_EXPAND_SUCCESS,
 } from '../actions/mutes';
+import {
+  DIRECTORY_FETCH_REQUEST,
+  DIRECTORY_FETCH_SUCCESS,
+  DIRECTORY_FETCH_FAIL,
+  DIRECTORY_EXPAND_REQUEST,
+  DIRECTORY_EXPAND_SUCCESS,
+  DIRECTORY_EXPAND_FAIL,
+} from 'mastodon/actions/directory';
 import { Map as ImmutableMap, List as ImmutableList } from 'immutable';
 
 const initialState = ImmutableMap({
@@ -74,6 +82,16 @@ export default function userLists(state = initialState, action) {
     return state.setIn(['mutes', 'items'], ImmutableList(action.accounts.map(item => item.id))).setIn(['mutes', 'next'], action.next);
   case MUTES_EXPAND_SUCCESS:
     return state.updateIn(['mutes', 'items'], list => list.concat(action.accounts.map(item => item.id))).setIn(['mutes', 'next'], action.next);
+  case DIRECTORY_FETCH_SUCCESS:
+    return state.setIn(['directory', 'items'], ImmutableList(action.accounts.map(item => item.id))).setIn(['directory', 'isLoading'], false);
+  case DIRECTORY_EXPAND_SUCCESS:
+    return state.updateIn(['directory', 'items'], list => list.concat(action.accounts.map(item => item.id))).setIn(['directory', 'isLoading'], false);
+  case DIRECTORY_FETCH_REQUEST:
+  case DIRECTORY_EXPAND_REQUEST:
+    return state.setIn(['directory', 'isLoading'], true);
+  case DIRECTORY_FETCH_FAIL:
+  case DIRECTORY_EXPAND_FAIL:
+    return state.setIn(['directory', 'isLoading'], false);
   default:
     return state;
   }
diff --git a/app/javascript/styles/mastodon/components.scss b/app/javascript/styles/mastodon/components.scss
index 36466d5c1..1129680f1 100644
--- a/app/javascript/styles/mastodon/components.scss
+++ b/app/javascript/styles/mastodon/components.scss
@@ -2092,13 +2092,23 @@ a.account__display-name {
     padding: 0;
   }
 
-  //.column {
-  //  margin-top: 0;
+  .directory__list {
+    display: grid;
+    grid-gap: 10px;
+    grid-template-columns: minmax(0, 50%) minmax(0, 50%);
 
-  //  @media screen and (min-width: $no-gap-breakpoint) {
-  //    margin-top: 10px;
-  //  }
-  //}
+    @media screen and (max-width: $no-gap-breakpoint) {
+      display: block;
+    }
+  }
+
+  .directory__card {
+    margin-bottom: 0;
+  }
+
+  .filter-form {
+    display: flex;
+  }
 
   .autosuggest-textarea__textarea {
     font-size: 16px;
@@ -4982,59 +4992,6 @@ a.status-card.compact:hover {
 }
 /* End Media Gallery */
 
-/* Status Video Player */
-.status__video-player {
-  background: $base-overlay-background;
-  box-sizing: border-box;
-  cursor: default; /* May not be needed */
-  margin-top: 8px;
-  overflow: hidden;
-  position: relative;
-}
-
-.status__video-player-video {
-  height: 100%;
-  object-fit: cover;
-  position: relative;
-  top: 50%;
-  transform: translateY(-50%);
-  width: 100%;
-  z-index: 1;
-}
-
-.status__video-player-expand,
-.status__video-player-mute {
-  color: $primary-text-color;
-  opacity: 0.8;
-  position: absolute;
-  right: 4px;
-  text-shadow: 0 1px 1px $base-shadow-color, 1px 0 1px $base-shadow-color;
-}
-
-.status__video-player-spoiler {
-  display: none;
-  color: $primary-text-color;
-  left: 4px;
-  position: absolute;
-  text-shadow: 0 1px 1px $base-shadow-color, 1px 0 1px $base-shadow-color;
-  top: 4px;
-  z-index: 100;
-
-  &.status__video-player-spoiler--visible {
-    display: block;
-  }
-}
-
-.status__video-player-expand {
-  bottom: 4px;
-  z-index: 100;
-}
-
-.status__video-player-mute {
-  top: 4px;
-  z-index: 5;
-}
-
 .detailed,
 .fullscreen {
   .video-player__volume__current,
@@ -5387,28 +5344,130 @@ a.status-card.compact:hover {
   }
 }
 
-.media-spoiler-video {
-  background-size: cover;
-  background-repeat: no-repeat;
-  background-position: center;
-  cursor: pointer;
-  margin-top: 8px;
-  position: relative;
-  border: 0;
-  display: block;
-}
+.directory {
+  &__list {
+    width: 100%;
+    margin: 10px 0;
+    transition: opacity 100ms ease-in;
 
-.media-spoiler-video-play-icon {
-  border-radius: 100px;
-  color: rgba($primary-text-color, 0.8);
-  font-size: 36px;
-  left: 50%;
-  padding: 5px;
-  position: absolute;
-  top: 50%;
-  transform: translate(-50%, -50%);
+    &.loading {
+      opacity: 0.7;
+    }
+
+    @media screen and (max-width: $no-gap-breakpoint) {
+      margin: 0;
+    }
+  }
+
+  &__card {
+    box-sizing: border-box;
+    margin-bottom: 10px;
+
+    &__img {
+      height: 125px;
+      position: relative;
+      background: darken($ui-base-color, 12%);
+
+      img {
+        display: block;
+        width: 100%;
+        height: 100%;
+        margin: 0;
+        object-fit: cover;
+      }
+    }
+
+    &__bar {
+      display: flex;
+      align-items: center;
+      background: lighten($ui-base-color, 4%);
+      padding: 10px;
+
+      &__name {
+        flex: 1 1 auto;
+        display: flex;
+        align-items: center;
+        text-decoration: none;
+      }
+
+      &__relationship {
+        width: 23px;
+        min-height: 1px;
+        flex: 0 0 auto;
+      }
+
+      .avatar {
+        flex: 0 0 auto;
+        width: 48px;
+        height: 48px;
+        padding-top: 2px;
+
+        img {
+          width: 100%;
+          height: 100%;
+          display: block;
+          margin: 0;
+          border-radius: 4px;
+          background: darken($ui-base-color, 8%);
+          object-fit: cover;
+        }
+      }
+
+      .display-name {
+        margin-left: 15px;
+        text-align: left;
+
+        strong {
+          font-size: 15px;
+          color: $primary-text-color;
+          font-weight: 500;
+          overflow: hidden;
+          text-overflow: ellipsis;
+        }
+
+        span {
+          display: block;
+          font-size: 14px;
+          color: $darker-text-color;
+          font-weight: 400;
+          overflow: hidden;
+          text-overflow: ellipsis;
+        }
+      }
+    }
+
+    &__extra {
+      background: $ui-base-color;
+      display: flex;
+      align-items: center;
+      justify-content: center;
+
+      .accounts-table__count {
+        width: 33.33%;
+        flex: 0 0 auto;
+        padding: 15px 0;
+      }
+
+      .account__header__content {
+        box-sizing: border-box;
+        padding: 15px 10px;
+        border-bottom: 1px solid lighten($ui-base-color, 8%);
+        width: 100%;
+        white-space: nowrap;
+        overflow: hidden;
+        text-overflow: ellipsis;
+
+        p {
+          display: none;
+
+          &:first-child {
+            display: inline;
+          }
+        }
+      }
+    }
+  }
 }
-/* End Video Player */
 
 .account-gallery__container {
   display: flex;
@@ -5484,6 +5543,73 @@ a.status-card.compact:hover {
       }
     }
   }
+
+  &.directory__section-headline {
+    background: darken($ui-base-color, 2%);
+    border-bottom-color: transparent;
+
+    a,
+    button {
+      &.active {
+        &::before {
+          display: none;
+        }
+
+        &::after {
+          border-color: transparent transparent darken($ui-base-color, 7%);
+        }
+      }
+    }
+  }
+}
+
+.filter-form {
+  background: $ui-base-color;
+
+  &__column {
+    padding: 10px 15px;
+  }
+
+  .radio-button {
+    display: block;
+  }
+}
+
+.radio-button {
+  font-size: 14px;
+  position: relative;
+  display: inline-block;
+  padding: 6px 0;
+  line-height: 18px;
+  cursor: default;
+  white-space: nowrap;
+  overflow: hidden;
+  text-overflow: ellipsis;
+  cursor: pointer;
+
+  input[type=radio],
+  input[type=checkbox] {
+    display: none;
+  }
+
+  &__input {
+    display: inline-block;
+    position: relative;
+    border: 1px solid $ui-primary-color;
+    box-sizing: border-box;
+    width: 18px;
+    height: 18px;
+    flex: 0 0 auto;
+    margin-right: 10px;
+    top: -1px;
+    border-radius: 50%;
+    vertical-align: middle;
+
+    &.checked {
+      border-color: lighten($ui-highlight-color, 8%);
+      background: lighten($ui-highlight-color, 8%);
+    }
+  }
 }
 
 ::-webkit-scrollbar-thumb {
diff --git a/app/lib/activitypub/adapter.rb b/app/lib/activitypub/adapter.rb
index a1d84de2f..1c58be8c0 100644
--- a/app/lib/activitypub/adapter.rb
+++ b/app/lib/activitypub/adapter.rb
@@ -20,6 +20,7 @@ class ActivityPub::Adapter < ActiveModelSerializers::Adapter::Base
     focal_point: { 'toot' => 'http://joinmastodon.org/ns#', 'focalPoint' => { '@container' => '@list', '@id' => 'toot:focalPoint' } },
     identity_proof: { 'toot' => 'http://joinmastodon.org/ns#', 'IdentityProof' => 'toot:IdentityProof' },
     blurhash: { 'toot' => 'http://joinmastodon.org/ns#', 'blurhash' => 'toot:blurhash' },
+    discoverable: { 'toot' => 'http://joinmastodon.org/ns#', 'discoverable' => 'toot:discoverable' },
   }.freeze
 
   def self.default_key_transform
diff --git a/app/models/account.rb b/app/models/account.rb
index 392cc625f..8c9388b95 100644
--- a/app/models/account.rb
+++ b/app/models/account.rb
@@ -51,7 +51,6 @@
 class Account < ApplicationRecord
   USERNAME_RE = /[a-z0-9_]+([a-z0-9_\.-]+[a-z0-9_]+)?/i
   MENTION_RE  = /(?<=^|[^\/[:word:]])@((#{USERNAME_RE})(?:@[a-z0-9\.\-]+[a-z0-9]+)?)/i
-  MIN_FOLLOWERS_DISCOVERY = 10
 
   include AccountAssociations
   include AccountAvatar
@@ -100,11 +99,13 @@ class Account < ApplicationRecord
   scope :matches_display_name, ->(value) { where(arel_table[:display_name].matches("#{value}%")) }
   scope :matches_domain, ->(value) { where(arel_table[:domain].matches("%#{value}%")) }
   scope :searchable, -> { without_suspended.where(moved_to_account_id: nil) }
-  scope :discoverable, -> { searchable.without_silenced.where(discoverable: true).joins(:account_stat).where(AccountStat.arel_table[:followers_count].gteq(MIN_FOLLOWERS_DISCOVERY)) }
+  scope :discoverable, -> { searchable.without_silenced.where(discoverable: true).left_outer_joins(:account_stat) }
   scope :tagged_with, ->(tag) { joins(:accounts_tags).where(accounts_tags: { tag_id: tag }) }
-  scope :by_recent_status, -> { order(Arel.sql('(case when account_stats.last_status_at is null then 1 else 0 end) asc, account_stats.last_status_at desc')) }
+  scope :by_recent_status, -> { order(Arel.sql('(case when account_stats.last_status_at is null then 1 else 0 end) asc, account_stats.last_status_at desc, accounts.id desc')) }
   scope :popular, -> { order('account_stats.followers_count desc') }
   scope :by_domain_and_subdomains, ->(domain) { where(domain: domain).or(where(arel_table[:domain].matches('%.' + domain))) }
+  scope :not_excluded_by_account, ->(account) { where.not(id: account.excluded_from_timeline_account_ids) }
+  scope :not_domain_blocked_by_account, ->(account) { where(arel_table[:domain].eq(nil).or(arel_table[:domain].not_in(account.excluded_from_timeline_domains))) }
 
   delegate :email,
            :unconfirmed_email,
diff --git a/app/serializers/activitypub/actor_serializer.rb b/app/serializers/activitypub/actor_serializer.rb
index 0bd7aed2e..222e17c99 100644
--- a/app/serializers/activitypub/actor_serializer.rb
+++ b/app/serializers/activitypub/actor_serializer.rb
@@ -6,12 +6,14 @@ class ActivityPub::ActorSerializer < ActivityPub::Serializer
   context :security
 
   context_extensions :manually_approves_followers, :featured, :also_known_as,
-                     :moved_to, :property_value, :hashtag, :emoji, :identity_proof
+                     :moved_to, :property_value, :hashtag, :emoji, :identity_proof,
+                     :discoverable
 
   attributes :id, :type, :following, :followers,
              :inbox, :outbox, :featured,
              :preferred_username, :name, :summary,
-             :url, :manually_approves_followers
+             :url, :manually_approves_followers,
+             :discoverable
 
   has_one :public_key, serializer: ActivityPub::PublicKeySerializer
 
diff --git a/app/serializers/rest/account_serializer.rb b/app/serializers/rest/account_serializer.rb
index 272e3eb9c..75b6cf13b 100644
--- a/app/serializers/rest/account_serializer.rb
+++ b/app/serializers/rest/account_serializer.rb
@@ -5,7 +5,7 @@ class REST::AccountSerializer < ActiveModel::Serializer
 
   attributes :id, :username, :acct, :display_name, :locked, :bot, :created_at,
              :note, :url, :avatar, :avatar_static, :header, :header_static,
-             :followers_count, :following_count, :statuses_count
+             :followers_count, :following_count, :statuses_count, :last_status_at
 
   has_one :moved_to_account, key: :moved, serializer: REST::AccountSerializer, if: :moved_and_not_nested?
   has_many :emojis, serializer: REST::CustomEmojiSerializer
diff --git a/app/services/activitypub/process_account_service.rb b/app/services/activitypub/process_account_service.rb
index 603e27ed9..cef658e19 100644
--- a/app/services/activitypub/process_account_service.rb
+++ b/app/services/activitypub/process_account_service.rb
@@ -83,6 +83,7 @@ class ActivityPub::ProcessAccountService < BaseService
     @account.fields                  = property_values || {}
     @account.also_known_as           = as_array(@json['alsoKnownAs'] || []).map { |item| value_or_id(item) }
     @account.actor_type              = actor_type
+    @account.discoverable            = @json['discoverable'] || false
   end
 
   def set_fetchable_attributes!
diff --git a/app/views/application/_card.html.haml b/app/views/application/_card.html.haml
index 00254c40c..8719ce484 100644
--- a/app/views/application/_card.html.haml
+++ b/app/views/application/_card.html.haml
@@ -9,7 +9,7 @@
         = image_tag account.avatar.url, alt: '', width: 48, height: 48, class: 'u-photo'
 
       .display-name
-        %span{id: "default_account_display_name", style: "display:none;"}= account.username
+        %span{ id: "default_account_display_name", style: "display: none" }= account.username
         %bdi
           %strong.emojify.p-name= display_name(account, custom_emojify: true)
         %span
diff --git a/app/views/directories/index.html.haml b/app/views/directories/index.html.haml
index a8aa68cc4..54b27114c 100644
--- a/app/views/directories/index.html.haml
+++ b/app/views/directories/index.html.haml
@@ -14,58 +14,10 @@
   %h1= t('directories.explore_mastodon', title: site_title)
   %p= t('directories.explanation')
 
-.grid
-  .column-0
-    - if @accounts.empty?
-      = nothing_here
-    - else
-      .directory
-        %table.accounts-table
-          %tbody
-            - @accounts.each do |account|
-              %tr
-                %td= account_link_to account
-                %td.accounts-table__count.optional
-                  = number_to_human account.statuses_count, strip_insignificant_zeros: true
-                  %small= t('accounts.posts', count: account.statuses_count).downcase
-                %td.accounts-table__count.optional
-                  = number_to_human account.followers_count, strip_insignificant_zeros: true
-                  %small= t('accounts.followers', count: account.followers_count).downcase
-                %td.accounts-table__count
-                  - if account.last_status_at.present?
-                    %time.time-ago{ datetime: account.last_status_at.iso8601, title: l(account.last_status_at) }= l account.last_status_at
-                  - else
-                    \-
-                  %small= t('accounts.last_active')
+- if @accounts.empty?
+  = nothing_here
+- else
+  .card-grid
+    = render partial: 'application/card', collection: @accounts, as: :account
 
-      = paginate @accounts
-
-  .column-1
-    - if user_signed_in?
-      .box-widget.notice-widget
-        - if current_account.discoverable?
-          - if current_account.followers_count < Account::MIN_FOLLOWERS_DISCOVERY
-            %p= t('directories.enabled_but_waiting', min_followers: Account::MIN_FOLLOWERS_DISCOVERY)
-          - else
-            %p= t('directories.enabled')
-        - else
-          %p= t('directories.how_to_enable')
-
-          = link_to settings_profile_path do
-            = t('settings.edit_profile')
-            = fa_icon 'chevron-right fw'
-
-    - if @tags.empty? && !user_signed_in?
-      .nothing-here
-    - else
-      - @tags.each do |tag|
-        .directory__tag{ class: tag.id == @tag&.id ? 'active' : nil }
-          = link_to explore_hashtag_path(tag) do
-            %h4
-              = fa_icon 'hashtag'
-              = tag.name
-              %small= t('directories.people', count: tag.accounts_count)
-
-            .avatar-stack
-              - tag.cached_sample_accounts.each do |account|
-                = image_tag current_account&.user&.setting_auto_play_gif ? account.avatar_original_url : account.avatar_static_url, width: 48, height: 48, alt: '', class: 'account__avatar'
+  = paginate @accounts
diff --git a/app/views/settings/profiles/show.html.haml b/app/views/settings/profiles/show.html.haml
index f8a8fddd3..f042011d6 100644
--- a/app/views/settings/profiles/show.html.haml
+++ b/app/views/settings/profiles/show.html.haml
@@ -28,7 +28,7 @@
 
   - if Setting.profile_directory
     .fields-group
-      = f.input :discoverable, as: :boolean, wrapper: :with_label, hint: t('simple_form.hints.defaults.discoverable_html', min_followers: Account::MIN_FOLLOWERS_DISCOVERY, path: explore_path), recommended: true
+      = f.input :discoverable, as: :boolean, wrapper: :with_label, hint: t('simple_form.hints.defaults.discoverable'), recommended: true
 
   %hr.spacer/
 
diff --git a/config/locales/en.yml b/config/locales/en.yml
index ee78e4720..2f601f274 100644
--- a/config/locales/en.yml
+++ b/config/locales/en.yml
@@ -630,14 +630,8 @@ en:
     warning_title: Disseminated content availability
   directories:
     directory: Profile directory
-    enabled: You are currently listed in the directory.
-    enabled_but_waiting: You have opted-in to be listed in the directory, but you do not have the minimum number of followers (%{min_followers}) to be listed yet.
     explanation: Discover users based on their interests
     explore_mastodon: Explore %{title}
-    how_to_enable: You are not currently opted-in to the directory. You can opt-in below. Use hashtags in your bio text to be listed under specific hashtags!
-    people:
-      one: "%{count} person"
-      other: "%{count} people"
   domain_blocks:
     blocked_domains: List of limited and blocked domains
     description: This is the list of servers that %{instance} limits or reject federation with.
diff --git a/config/locales/simple_form.en.yml b/config/locales/simple_form.en.yml
index 2d5ada0a4..2e5982de9 100644
--- a/config/locales/simple_form.en.yml
+++ b/config/locales/simple_form.en.yml
@@ -16,7 +16,7 @@ en:
         bot: This account mainly performs automated actions and might not be monitored
         context: One or multiple contexts where the filter should apply
         digest: Only sent after a long period of inactivity and only if you have received any personal messages in your absence
-        discoverable_html: The <a href="%{path}" target="_blank">directory</a> lets people find accounts based on interests and activity. Requires at least %{min_followers} followers
+        discoverable: The profile directory is another way by which your account can reach a wider audience
         email: You will be sent a confirmation e-mail
         fields: You can have up to 4 items displayed as a table on your profile
         header: PNG, GIF or JPG. At most %{size}. Will be downscaled to %{dimensions}px
diff --git a/config/routes.rb b/config/routes.rb
index 9ae24b0cd..92f272ff5 100644
--- a/config/routes.rb
+++ b/config/routes.rb
@@ -325,6 +325,7 @@ Rails.application.routes.draw do
       end
 
       resource :domain_blocks, only: [:show, :create, :destroy]
+      resource :directory, only: [:show]
 
       resources :follow_requests, only: [:index] do
         member do
-- 
cgit 


From b54b725d6bc8dd0a4ab0fe0bf408193c1bae8106 Mon Sep 17 00:00:00 2001
From: Eugen Rochko <eugen@zeonfederated.com>
Date: Fri, 30 Aug 2019 02:19:17 +0200
Subject: Fix uncaught domain normalization error in remote follow (#11703)

---
 app/controllers/remote_follow_controller.rb       |  2 +-
 app/controllers/remote_interaction_controller.rb  |  2 +-
 app/models/remote_follow.rb                       |  6 ++++--
 app/validators/domain_validator.rb                | 12 ++++++++++--
 spec/controllers/remote_follow_controller_spec.rb |  4 +---
 spec/models/remote_follow_spec.rb                 |  2 +-
 6 files changed, 18 insertions(+), 10 deletions(-)

(limited to 'app/models')

diff --git a/app/controllers/remote_follow_controller.rb b/app/controllers/remote_follow_controller.rb
index 0fb71d335..ba963a7a0 100644
--- a/app/controllers/remote_follow_controller.rb
+++ b/app/controllers/remote_follow_controller.rb
@@ -29,7 +29,7 @@ class RemoteFollowController < ApplicationController
   end
 
   def session_params
-    { acct: session[:remote_follow] }
+    { acct: session[:remote_follow] || current_account&.username }
   end
 
   def set_body_classes
diff --git a/app/controllers/remote_interaction_controller.rb b/app/controllers/remote_interaction_controller.rb
index de5616e25..15224e853 100644
--- a/app/controllers/remote_interaction_controller.rb
+++ b/app/controllers/remote_interaction_controller.rb
@@ -32,7 +32,7 @@ class RemoteInteractionController < ApplicationController
   end
 
   def session_params
-    { acct: session[:remote_follow] }
+    { acct: session[:remote_follow] || current_account&.username }
   end
 
   def set_status
diff --git a/app/models/remote_follow.rb b/app/models/remote_follow.rb
index 93df11724..52dd3f67b 100644
--- a/app/models/remote_follow.rb
+++ b/app/models/remote_follow.rb
@@ -6,7 +6,7 @@ class RemoteFollow
 
   attr_accessor :acct, :addressable_template
 
-  validates :acct, presence: true
+  validates :acct, presence: true, domain: { acct: true }
 
   def initialize(attrs = {})
     @acct = normalize_acct(attrs[:acct])
@@ -21,7 +21,7 @@ class RemoteFollow
   end
 
   def subscribe_address_for(account)
-    addressable_template.expand(uri: account.local_username_and_domain).to_s
+    addressable_template.expand(uri: ActivityPub::TagManager.instance.uri_for(account)).to_s
   end
 
   def interact_address_for(status)
@@ -44,6 +44,8 @@ class RemoteFollow
     end
 
     [username, domain].compact.join('@')
+  rescue Addressable::URI::InvalidURIError
+    value
   end
 
   def fetch_template!
diff --git a/app/validators/domain_validator.rb b/app/validators/domain_validator.rb
index ae07f1798..6e4a854ff 100644
--- a/app/validators/domain_validator.rb
+++ b/app/validators/domain_validator.rb
@@ -4,14 +4,22 @@ class DomainValidator < ActiveModel::EachValidator
   def validate_each(record, attribute, value)
     return if value.blank?
 
-    record.errors.add(attribute, I18n.t('domain_validator.invalid_domain')) unless compliant?(value)
+    domain = begin
+      if options[:acct]
+        value.split('@').last
+      else
+        value
+      end
+    end
+
+    record.errors.add(attribute, I18n.t('domain_validator.invalid_domain')) unless compliant?(domain)
   end
 
   private
 
   def compliant?(value)
     Addressable::URI.new.tap { |uri| uri.host = value }
-  rescue Addressable::URI::InvalidURIError
+  rescue Addressable::URI::InvalidURIError, IDN::Idna::IdnaError
     false
   end
 end
diff --git a/spec/controllers/remote_follow_controller_spec.rb b/spec/controllers/remote_follow_controller_spec.rb
index 5088c2e65..d79dd2949 100644
--- a/spec/controllers/remote_follow_controller_spec.rb
+++ b/spec/controllers/remote_follow_controller_spec.rb
@@ -66,9 +66,7 @@ describe RemoteFollowController do
         end
 
         it 'redirects to the remote location' do
-          address = "http://example.com/follow_me?acct=test_user%40#{Rails.configuration.x.local_domain}"
-
-          expect(response).to redirect_to(address)
+          expect(response).to redirect_to("http://example.com/follow_me?acct=https%3A%2F%2F#{Rails.configuration.x.local_domain}%2Fusers%2Ftest_user")
         end
       end
     end
diff --git a/spec/models/remote_follow_spec.rb b/spec/models/remote_follow_spec.rb
index ed2667b28..5b4c19b5b 100644
--- a/spec/models/remote_follow_spec.rb
+++ b/spec/models/remote_follow_spec.rb
@@ -61,7 +61,7 @@ RSpec.describe RemoteFollow do
     subject { remote_follow.subscribe_address_for(account) }
 
     it 'returns subscribe address' do
-      is_expected.to eq 'https://quitter.no/main/ostatussub?profile=alice%40cb6e6126.ngrok.io'
+      is_expected.to eq 'https://quitter.no/main/ostatussub?profile=https%3A%2F%2Fcb6e6126.ngrok.io%2Fusers%2Falice'
     end
   end
 end
-- 
cgit 


From 5f591d979c12984f8631eb60bcaeb461d3a097a3 Mon Sep 17 00:00:00 2001
From: Eugen Rochko <eugen@zeonfederated.com>
Date: Fri, 30 Aug 2019 02:49:54 +0200
Subject: Fix non-numbers being passed to Redis and causing an error (#11697)

---
 app/models/feed.rb | 5 +++++
 1 file changed, 5 insertions(+)

(limited to 'app/models')

diff --git a/app/models/feed.rb b/app/models/feed.rb
index 0e8943ff8..36e0c1e0a 100644
--- a/app/models/feed.rb
+++ b/app/models/feed.rb
@@ -9,6 +9,11 @@ class Feed
   end
 
   def get(limit, max_id = nil, since_id = nil, min_id = nil)
+    limit    = limit.to_i
+    max_id   = max_id.to_i if max_id.present?
+    since_id = since_id.to_i if since_id.present?
+    min_id   = min_id.to_i if min_id.present?
+
     from_redis(limit, max_id, since_id, min_id)
   end
 
-- 
cgit 


From 7235f538c6b6f6ddcbc9c35af16e96f266485712 Mon Sep 17 00:00:00 2001
From: umonaca <53662960+umonaca@users.noreply.github.com>
Date: Thu, 29 Aug 2019 22:30:29 -0400
Subject: Add more media formats (#11687)

* Add more media formats

* Removed amr file type
---
 app/models/media_attachment.rb | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

(limited to 'app/models')

diff --git a/app/models/media_attachment.rb b/app/models/media_attachment.rb
index fe76b36aa..44f8e6be6 100644
--- a/app/models/media_attachment.rb
+++ b/app/models/media_attachment.rb
@@ -28,12 +28,12 @@ class MediaAttachment < ApplicationRecord
 
   IMAGE_FILE_EXTENSIONS = %w(.jpg .jpeg .png .gif).freeze
   VIDEO_FILE_EXTENSIONS = %w(.webm .mp4 .m4v .mov).freeze
-  AUDIO_FILE_EXTENSIONS = %w(.ogg .oga .mp3 .wav .flac .opus .aac .m4a .3gp).freeze
+  AUDIO_FILE_EXTENSIONS = %w(.ogg .oga .mp3 .wav .flac .opus .aac .m4a .3gp .wma).freeze
 
   IMAGE_MIME_TYPES             = %w(image/jpeg image/png image/gif).freeze
   VIDEO_MIME_TYPES             = %w(video/webm video/mp4 video/quicktime video/ogg).freeze
   VIDEO_CONVERTIBLE_MIME_TYPES = %w(video/webm video/quicktime).freeze
-  AUDIO_MIME_TYPES             = %w(audio/wave audio/wav audio/x-wav audio/x-pn-wave audio/ogg audio/mpeg audio/mp3 audio/webm audio/flac audio/aac audio/m4a audio/3gpp).freeze
+  AUDIO_MIME_TYPES             = %w(audio/wave audio/wav audio/x-wav audio/x-pn-wave audio/ogg audio/mpeg audio/mp3 audio/webm audio/flac audio/aac audio/m4a audio/x-m4a audio/mp4 audio/3gpp video/x-ms-asf).freeze
 
   BLURHASH_OPTIONS = {
     x_comp: 4,
-- 
cgit 


From 70ddef2654a931827ce5e4323e3042365f6078f2 Mon Sep 17 00:00:00 2001
From: Eugen Rochko <eugen@zeonfederated.com>
Date: Mon, 2 Sep 2019 18:11:13 +0200
Subject: Change trending hashtags to not disappear instantly after midnight
 (#11712)

---
 app/controllers/admin/tags_controller.rb           |   2 +-
 app/lib/feed_manager.rb                            |   2 +-
 app/models/tag.rb                                  |   4 +-
 app/models/trending_tags.rb                        | 102 +++++++++++++++------
 app/workers/scheduler/trending_tags_scheduler.rb   |  11 +++
 config/sidekiq.yml                                 |   3 +
 db/migrate/20190901035623_add_max_score_to_tags.rb |   6 ++
 .../20190901040524_remove_score_from_tags.rb       |  12 +++
 db/schema.rb                                       |   6 +-
 spec/models/trending_tags_spec.rb                  |  68 ++++++++++++++
 10 files changed, 179 insertions(+), 37 deletions(-)
 create mode 100644 app/workers/scheduler/trending_tags_scheduler.rb
 create mode 100644 db/migrate/20190901035623_add_max_score_to_tags.rb
 create mode 100644 db/post_migrate/20190901040524_remove_score_from_tags.rb
 create mode 100644 spec/models/trending_tags_spec.rb

(limited to 'app/models')

diff --git a/app/controllers/admin/tags_controller.rb b/app/controllers/admin/tags_controller.rb
index 25d9b7d3d..8bd4e5f8b 100644
--- a/app/controllers/admin/tags_controller.rb
+++ b/app/controllers/admin/tags_controller.rb
@@ -57,7 +57,7 @@ module Admin
       scope = scope.unreviewed if filter_params[:review] == 'unreviewed'
       scope = scope.reviewed.order(reviewed_at: :desc) if filter_params[:review] == 'reviewed'
       scope = scope.pending_review.order(requested_review_at: :desc) if filter_params[:review] == 'pending_review'
-      scope.order(score: :desc)
+      scope.order(max_score: :desc)
     end
 
     def filter_params
diff --git a/app/lib/feed_manager.rb b/app/lib/feed_manager.rb
index ca3d890a8..871ec5c19 100644
--- a/app/lib/feed_manager.rb
+++ b/app/lib/feed_manager.rb
@@ -63,7 +63,7 @@ class FeedManager
     reblog_key   = key(type, account_id, 'reblogs')
 
     # Remove any items past the MAX_ITEMS'th entry in our feed
-    redis.zremrangebyrank(timeline_key, '0', (-(FeedManager::MAX_ITEMS + 1)).to_s)
+    redis.zremrangebyrank(timeline_key, 0, -(FeedManager::MAX_ITEMS + 1))
 
     # Get the score of the REBLOG_FALLOFF'th item in our feed, and stop
     # tracking anything after it for deduplication purposes.
diff --git a/app/models/tag.rb b/app/models/tag.rb
index 945e3a3c6..135e0a030 100644
--- a/app/models/tag.rb
+++ b/app/models/tag.rb
@@ -7,14 +7,14 @@
 #  name                :string           default(""), not null
 #  created_at          :datetime         not null
 #  updated_at          :datetime         not null
-#  score               :integer
 #  usable              :boolean
 #  trendable           :boolean
 #  listable            :boolean
 #  reviewed_at         :datetime
 #  requested_review_at :datetime
 #  last_status_at      :datetime
-#  last_trend_at       :datetime
+#  max_score           :float
+#  max_score_at        :datetime
 #
 
 class Tag < ApplicationRecord
diff --git a/app/models/trending_tags.rb b/app/models/trending_tags.rb
index e4ce988c1..e1b92b175 100644
--- a/app/models/trending_tags.rb
+++ b/app/models/trending_tags.rb
@@ -7,6 +7,8 @@ class TrendingTags
   THRESHOLD            = 5
   LIMIT                = 10
   REVIEW_THRESHOLD     = 3
+  MAX_SCORE_COOLDOWN   = 3.days.freeze
+  MAX_SCORE_HALFLIFE   = 6.hours.freeze
 
   class << self
     include Redisable
@@ -16,14 +18,75 @@ class TrendingTags
 
       increment_historical_use!(tag.id, at_time)
       increment_unique_use!(tag.id, account.id, at_time)
-      increment_vote!(tag, at_time)
+      increment_use!(tag.id, at_time)
 
       tag.update(last_status_at: Time.now.utc) if tag.last_status_at.nil? || tag.last_status_at < 12.hours.ago
-      tag.update(last_trend_at: Time.now.utc)  if trending?(tag) && (tag.last_trend_at.nil? || tag.last_trend_at < 12.hours.ago)
+    end
+
+    def update!(at_time = Time.now.utc)
+      tag_ids = redis.smembers("#{KEY}:used:#{at_time.beginning_of_day.to_i}") + redis.zrange(KEY, 0, -1)
+      tags    = Tag.where(id: tag_ids.uniq)
+
+      # First pass to calculate scores and update the set
+
+      tags.each do |tag|
+        expected  = redis.pfcount("activity:tags:#{tag.id}:#{(at_time - 1.day).beginning_of_day.to_i}:accounts").to_f
+        expected  = 1.0 if expected.zero?
+        observed  = redis.pfcount("activity:tags:#{tag.id}:#{at_time.beginning_of_day.to_i}:accounts").to_f
+        max_time  = tag.max_score_at
+        max_score = tag.max_score
+        max_score = 0 if max_time.nil? || max_time < (at_time - MAX_SCORE_COOLDOWN)
+
+        score = begin
+          if expected > observed || observed < THRESHOLD
+            0
+          else
+            ((observed - expected)**2) / expected
+          end
+        end
+
+        if score > max_score
+          max_score = score
+          max_time  = at_time
+
+          # Not interested in triggering any callbacks for this
+          tag.update_columns(max_score: max_score, max_score_at: max_time)
+        end
+
+        decaying_score = max_score * (0.5**((at_time.to_f - max_time.to_f) / MAX_SCORE_HALFLIFE.to_f))
+
+        if decaying_score.zero?
+          redis.zrem(KEY, tag.id)
+        else
+          redis.zadd(KEY, decaying_score, tag.id)
+        end
+      end
+
+      users_for_review = User.staff.includes(:account).to_a.select(&:allows_trending_tag_emails?)
+
+      # Second pass to notify about previously unreviewed trends
+
+      tags.each do |tag|
+        current_rank              = redis.zrevrank(KEY, tag.id)
+        needs_review_notification = tag.requires_review? && !tag.requested_review?
+        rank_passes_threshold     = current_rank.present? && current_rank <= REVIEW_THRESHOLD
+
+        next unless !tag.trendable? && rank_passes_threshold && needs_review_notification
+
+        tag.touch(:requested_review_at)
+
+        users_for_review.each do |user|
+          AdminMailer.new_trending_tag(user.account, tag).deliver_later!
+        end
+      end
+
+      # Trim older items
+
+      redis.zremrangebyrank(KEY, 0, -(LIMIT + 1))
     end
 
     def get(limit, filtered: true)
-      tag_ids = redis.zrevrange("#{KEY}:#{Time.now.utc.beginning_of_day.to_i}", 0, LIMIT - 1).map(&:to_i)
+      tag_ids = redis.zrevrange(KEY, 0, LIMIT - 1).map(&:to_i)
 
       tags = Tag.where(id: tag_ids)
       tags = tags.where(trendable: true) if filtered
@@ -33,8 +96,8 @@ class TrendingTags
     end
 
     def trending?(tag)
-      rank = redis.zrevrank("#{KEY}:#{Time.now.utc.beginning_of_day.to_i}", tag.id)
-      rank.present? && rank <= LIMIT
+      rank = redis.zrevrank(KEY, tag.id)
+      rank.present? && rank < LIMIT
     end
 
     private
@@ -51,31 +114,10 @@ class TrendingTags
       redis.expire(key, EXPIRE_HISTORY_AFTER)
     end
 
-    def increment_vote!(tag, at_time)
-      key      = "#{KEY}:#{at_time.beginning_of_day.to_i}"
-      expected = redis.pfcount("activity:tags:#{tag.id}:#{(at_time - 1.day).beginning_of_day.to_i}:accounts").to_f
-      expected = 1.0 if expected.zero?
-      observed = redis.pfcount("activity:tags:#{tag.id}:#{at_time.beginning_of_day.to_i}:accounts").to_f
-
-      if expected > observed || observed < THRESHOLD
-        redis.zrem(key, tag.id)
-      else
-        score    = ((observed - expected)**2) / expected
-        old_rank = redis.zrevrank(key, tag.id)
-
-        redis.zadd(key, score, tag.id)
-        request_review!(tag) if (old_rank.nil? || old_rank > REVIEW_THRESHOLD) && redis.zrevrank(key, tag.id) <= REVIEW_THRESHOLD && !tag.trendable? && tag.requires_review? && !tag.requested_review?
-      end
-
-      redis.expire(key, EXPIRE_TRENDS_AFTER)
-    end
-
-    def request_review!(tag)
-      return unless Setting.trends
-
-      tag.touch(:requested_review_at)
-
-      User.staff.includes(:account).find_each { |u| AdminMailer.new_trending_tag(u.account, tag).deliver_later! if u.allows_trending_tag_emails? }
+    def increment_use!(tag_id, at_time)
+      key = "#{KEY}:used:#{at_time.beginning_of_day.to_i}"
+      redis.sadd(key, tag_id)
+      redis.expire(key, EXPIRE_HISTORY_AFTER)
     end
   end
 end
diff --git a/app/workers/scheduler/trending_tags_scheduler.rb b/app/workers/scheduler/trending_tags_scheduler.rb
new file mode 100644
index 000000000..77f0d5747
--- /dev/null
+++ b/app/workers/scheduler/trending_tags_scheduler.rb
@@ -0,0 +1,11 @@
+# frozen_string_literal: true
+
+class Scheduler::TrendingTagsScheduler
+  include Sidekiq::Worker
+
+  sidekiq_options unique: :until_executed, retry: 0
+
+  def perform
+    TrendingTags.update! if Setting.trends
+  end
+end
diff --git a/config/sidekiq.yml b/config/sidekiq.yml
index 6ebe450b0..5de25de23 100644
--- a/config/sidekiq.yml
+++ b/config/sidekiq.yml
@@ -9,6 +9,9 @@
   scheduled_statuses_scheduler:
     every: '5m'
     class: Scheduler::ScheduledStatusesScheduler
+  trending_tags_scheduler:
+    every: '5m'
+    class: Scheduler::TrendingTagsScheduler
   media_cleanup_scheduler:
     cron: '<%= Random.rand(0..59) %> <%= Random.rand(3..5) %> * * *'
     class: Scheduler::MediaCleanupScheduler
diff --git a/db/migrate/20190901035623_add_max_score_to_tags.rb b/db/migrate/20190901035623_add_max_score_to_tags.rb
new file mode 100644
index 000000000..f936e9871
--- /dev/null
+++ b/db/migrate/20190901035623_add_max_score_to_tags.rb
@@ -0,0 +1,6 @@
+class AddMaxScoreToTags < ActiveRecord::Migration[5.2]
+  def change
+    add_column :tags, :max_score, :float
+    add_column :tags, :max_score_at, :datetime
+  end
+end
diff --git a/db/post_migrate/20190901040524_remove_score_from_tags.rb b/db/post_migrate/20190901040524_remove_score_from_tags.rb
new file mode 100644
index 000000000..a1112700b
--- /dev/null
+++ b/db/post_migrate/20190901040524_remove_score_from_tags.rb
@@ -0,0 +1,12 @@
+# frozen_string_literal: true
+
+class RemoveScoreFromTags < ActiveRecord::Migration[5.2]
+  disable_ddl_transaction!
+
+  def change
+    safety_assured do
+      remove_column :tags, :score, :int
+      remove_column :tags, :last_trend_at, :datetime
+    end
+  end
+end
diff --git a/db/schema.rb b/db/schema.rb
index 482bca367..5576f70bf 100644
--- a/db/schema.rb
+++ b/db/schema.rb
@@ -10,7 +10,7 @@
 #
 # It's strongly recommended that you check this file into your version control system.
 
-ActiveRecord::Schema.define(version: 2019_08_23_221802) do
+ActiveRecord::Schema.define(version: 2019_09_01_040524) do
 
   # These are extensions that must be enabled in order to support this database
   enable_extension "plpgsql"
@@ -664,14 +664,14 @@ ActiveRecord::Schema.define(version: 2019_08_23_221802) do
     t.string "name", default: "", null: false
     t.datetime "created_at", null: false
     t.datetime "updated_at", null: false
-    t.integer "score"
     t.boolean "usable"
     t.boolean "trendable"
     t.boolean "listable"
     t.datetime "reviewed_at"
     t.datetime "requested_review_at"
     t.datetime "last_status_at"
-    t.datetime "last_trend_at"
+    t.float "max_score"
+    t.datetime "max_score_at"
     t.index "lower((name)::text)", name: "index_tags_on_name_lower", unique: true
   end
 
diff --git a/spec/models/trending_tags_spec.rb b/spec/models/trending_tags_spec.rb
new file mode 100644
index 000000000..b6122c994
--- /dev/null
+++ b/spec/models/trending_tags_spec.rb
@@ -0,0 +1,68 @@
+require 'rails_helper'
+
+RSpec.describe TrendingTags do
+  describe '.record_use!' do
+    pending
+  end
+
+  describe '.update!' do
+    let!(:at_time) { Time.now.utc }
+    let!(:tag1) { Fabricate(:tag, name: 'Catstodon') }
+    let!(:tag2) { Fabricate(:tag, name: 'DogsOfMastodon') }
+    let!(:tag3) { Fabricate(:tag, name: 'OCs') }
+
+    before do
+      allow(Redis.current).to receive(:pfcount) do |key|
+        case key
+        when "activity:tags:#{tag1.id}:#{(at_time - 1.day).beginning_of_day.to_i}:accounts"
+          2
+        when "activity:tags:#{tag1.id}:#{at_time.beginning_of_day.to_i}:accounts"
+          16
+        when "activity:tags:#{tag2.id}:#{(at_time - 1.day).beginning_of_day.to_i}:accounts"
+          0
+        when "activity:tags:#{tag2.id}:#{at_time.beginning_of_day.to_i}:accounts"
+          4
+        when "activity:tags:#{tag3.id}:#{(at_time - 1.day).beginning_of_day.to_i}:accounts"
+          13
+        end
+      end
+
+      Redis.current.zadd('trending_tags', 0.9, tag3.id)
+      Redis.current.sadd("trending_tags:used:#{at_time.beginning_of_day.to_i}", [tag1.id, tag2.id])
+
+      tag3.update(max_score: 0.9, max_score_at: (at_time - 1.day).beginning_of_day + 12.hours)
+
+      described_class.update!(at_time)
+    end
+
+    it 'calculates and re-calculates scores' do
+      expect(described_class.get(10, filtered: false)).to eq [tag1, tag3]
+    end
+
+    it 'omits hashtags below threshold' do
+      expect(described_class.get(10, filtered: false)).to_not include(tag2)
+    end
+
+    it 'decays scores' do
+      expect(Redis.current.zscore('trending_tags', tag3.id)).to be < 0.9
+    end
+  end
+
+  describe '.trending?' do
+    let(:tag) { Fabricate(:tag) }
+
+    before do
+      10.times { |i| Redis.current.zadd('trending_tags', i + 1, Fabricate(:tag).id) }
+    end
+
+    it 'returns true if the hashtag is within limit' do
+      Redis.current.zadd('trending_tags', 11, tag.id)
+      expect(described_class.trending?(tag)).to be true
+    end
+
+    it 'returns false if the hashtag is outside the limit' do
+      Redis.current.zadd('trending_tags', 0, tag.id)
+      expect(described_class.trending?(tag)).to be false
+    end
+  end
+end
-- 
cgit 


From e445a8af64908b2bdb721bec74c113e8258a129b Mon Sep 17 00:00:00 2001
From: Eugen Rochko <eugen@zeonfederated.com>
Date: Fri, 6 Sep 2019 13:55:51 +0200
Subject: Add timeline read markers API (#11762)

Fix #4093
---
 app/controllers/api/v1/markers_controller.rb       | 44 +++++++++++++++
 app/javascript/mastodon/actions/markers.js         | 30 ++++++++++
 app/javascript/mastodon/features/ui/index.js       |  5 +-
 app/models/marker.rb                               | 23 ++++++++
 app/models/user.rb                                 |  1 +
 app/serializers/rest/marker_serializer.rb          | 13 +++++
 config/routes.rb                                   |  1 +
 db/migrate/20190904222339_create_markers.rb        | 14 +++++
 db/schema.rb                                       | 14 ++++-
 spec/controllers/api/v1/markers_controller_spec.rb | 65 ++++++++++++++++++++++
 spec/fabricators/marker_fabricator.rb              |  6 ++
 spec/models/marker_spec.rb                         |  5 ++
 12 files changed, 219 insertions(+), 2 deletions(-)
 create mode 100644 app/controllers/api/v1/markers_controller.rb
 create mode 100644 app/javascript/mastodon/actions/markers.js
 create mode 100644 app/models/marker.rb
 create mode 100644 app/serializers/rest/marker_serializer.rb
 create mode 100644 db/migrate/20190904222339_create_markers.rb
 create mode 100644 spec/controllers/api/v1/markers_controller_spec.rb
 create mode 100644 spec/fabricators/marker_fabricator.rb
 create mode 100644 spec/models/marker_spec.rb

(limited to 'app/models')

diff --git a/app/controllers/api/v1/markers_controller.rb b/app/controllers/api/v1/markers_controller.rb
new file mode 100644
index 000000000..28c2ec791
--- /dev/null
+++ b/app/controllers/api/v1/markers_controller.rb
@@ -0,0 +1,44 @@
+# frozen_string_literal: true
+
+class Api::V1::MarkersController < Api::BaseController
+  before_action -> { doorkeeper_authorize! :read, :'read:statuses' }, only: [:index]
+  before_action -> { doorkeeper_authorize! :write, :'write:statuses' }, except: [:index]
+
+  before_action :require_user!
+
+  def index
+    @markers = current_user.markers.where(timeline: Array(params[:timeline])).each_with_object({}) { |marker, h| h[marker.timeline] = marker }
+    render json: serialize_map(@markers)
+  end
+
+  def create
+    Marker.transaction do
+      @markers = {}
+
+      resource_params.each_pair do |timeline, timeline_params|
+        @markers[timeline] = current_user.markers.find_or_initialize_by(timeline: timeline)
+        @markers[timeline].update!(timeline_params)
+      end
+    end
+
+    render json: serialize_map(@markers)
+  rescue ActiveRecord::StaleObjectError
+    render json: { error: 'Conflict during update, please try again' }, status: 409
+  end
+
+  private
+
+  def serialize_map(map)
+    serialized = {}
+
+    map.each_pair do |key, value|
+      serialized[key] = ActiveModelSerializers::SerializableResource.new(value, serializer: REST::MarkerSerializer).as_json
+    end
+
+    Oj.dump(serialized)
+  end
+
+  def resource_params
+    params.slice(*Marker::TIMELINES).permit(*Marker::TIMELINES.map { |timeline| { timeline.to_sym => [:last_read_id] } })
+  end
+end
diff --git a/app/javascript/mastodon/actions/markers.js b/app/javascript/mastodon/actions/markers.js
new file mode 100644
index 000000000..c3a5fe86f
--- /dev/null
+++ b/app/javascript/mastodon/actions/markers.js
@@ -0,0 +1,30 @@
+export const submitMarkers = () => (dispatch, getState) => {
+  const accessToken = getState().getIn(['meta', 'access_token'], '');
+  const params      = {};
+
+  const lastHomeId         = getState().getIn(['timelines', 'home', 'items', 0]);
+  const lastNotificationId = getState().getIn(['notifications', 'items', 0, 'id']);
+
+  if (lastHomeId) {
+    params.home = {
+      last_read_id: lastHomeId,
+    };
+  }
+
+  if (lastNotificationId) {
+    params.notifications = {
+      last_read_id: lastNotificationId,
+    };
+  }
+
+  if (Object.keys(params).length === 0) {
+    return;
+  }
+
+  const client = new XMLHttpRequest();
+
+  client.open('POST', '/api/v1/markers', false);
+  client.setRequestHeader('Content-Type', 'application/json');
+  client.setRequestHeader('Authorization', `Bearer ${accessToken}`);
+  client.send(JSON.stringify(params));
+};
diff --git a/app/javascript/mastodon/features/ui/index.js b/app/javascript/mastodon/features/ui/index.js
index 49c5c8d0e..63c5622b6 100644
--- a/app/javascript/mastodon/features/ui/index.js
+++ b/app/javascript/mastodon/features/ui/index.js
@@ -16,6 +16,7 @@ import { expandNotifications } from '../../actions/notifications';
 import { fetchFilters } from '../../actions/filters';
 import { clearHeight } from '../../actions/height_cache';
 import { focusApp, unfocusApp } from 'mastodon/actions/app';
+import { submitMarkers } from 'mastodon/actions/markers';
 import { WrappedSwitch, WrappedRoute } from './util/react_router_helpers';
 import UploadArea from './components/upload_area';
 import ColumnsAreaContainer from './containers/columns_area_container';
@@ -241,7 +242,9 @@ class UI extends React.PureComponent {
   };
 
   handleBeforeUnload = e => {
-    const { intl, isComposing, hasComposingText, hasMediaAttachments } = this.props;
+    const { intl, dispatch, isComposing, hasComposingText, hasMediaAttachments } = this.props;
+
+    dispatch(submitMarkers());
 
     if (isComposing && (hasComposingText || hasMediaAttachments)) {
       // Setting returnValue to any string causes confirmation dialog.
diff --git a/app/models/marker.rb b/app/models/marker.rb
new file mode 100644
index 000000000..a5bd2176a
--- /dev/null
+++ b/app/models/marker.rb
@@ -0,0 +1,23 @@
+# frozen_string_literal: true
+
+# == Schema Information
+#
+# Table name: markers
+#
+#  id           :bigint(8)        not null, primary key
+#  user_id      :bigint(8)
+#  timeline     :string           default(""), not null
+#  last_read_id :bigint(8)        default(0), not null
+#  lock_version :integer          default(0), not null
+#  created_at   :datetime         not null
+#  updated_at   :datetime         not null
+#
+
+class Marker < ApplicationRecord
+  TIMELINES = %w(home notifications).freeze
+
+  belongs_to :user
+
+  validates :timeline, :last_read_id, presence: true
+  validates :timeline, inclusion: { in: TIMELINES }
+end
diff --git a/app/models/user.rb b/app/models/user.rb
index a4a20d975..95f1d8fc5 100644
--- a/app/models/user.rb
+++ b/app/models/user.rb
@@ -74,6 +74,7 @@ class User < ApplicationRecord
   has_many :applications, class_name: 'Doorkeeper::Application', as: :owner
   has_many :backups, inverse_of: :user
   has_many :invites, inverse_of: :user
+  has_many :markers, inverse_of: :user, dependent: :destroy
 
   has_one :invite_request, class_name: 'UserInviteRequest', inverse_of: :user, dependent: :destroy
   accepts_nested_attributes_for :invite_request, reject_if: ->(attributes) { attributes['text'].blank? }
diff --git a/app/serializers/rest/marker_serializer.rb b/app/serializers/rest/marker_serializer.rb
new file mode 100644
index 000000000..2eaf3d507
--- /dev/null
+++ b/app/serializers/rest/marker_serializer.rb
@@ -0,0 +1,13 @@
+# frozen_string_literal: true
+
+class REST::MarkerSerializer < ActiveModel::Serializer
+  attributes :last_read_id, :version, :updated_at
+
+  def last_read_id
+    object.last_read_id.to_s
+  end
+
+  def version
+    object.lock_version
+  end
+end
diff --git a/config/routes.rb b/config/routes.rb
index c5326052e..74a162f32 100644
--- a/config/routes.rb
+++ b/config/routes.rb
@@ -314,6 +314,7 @@ Rails.application.routes.draw do
       resources :trends,       only: [:index]
       resources :filters,      only: [:index, :create, :show, :update, :destroy]
       resources :endorsements, only: [:index]
+      resources :markers,      only: [:index, :create]
 
       namespace :apps do
         get :verify_credentials, to: 'credentials#show'
diff --git a/db/migrate/20190904222339_create_markers.rb b/db/migrate/20190904222339_create_markers.rb
new file mode 100644
index 000000000..71ca70ac3
--- /dev/null
+++ b/db/migrate/20190904222339_create_markers.rb
@@ -0,0 +1,14 @@
+class CreateMarkers < ActiveRecord::Migration[5.2]
+  def change
+    create_table :markers do |t|
+      t.references :user, foreign_key: { on_delete: :cascade, index: false }
+      t.string :timeline, default: '', null: false
+      t.bigint :last_read_id, default: 0, null: false
+      t.integer :lock_version, default: 0, null: false
+
+      t.timestamps
+    end
+
+    add_index :markers, [:user_id, :timeline], unique: true
+  end
+end
diff --git a/db/schema.rb b/db/schema.rb
index 5576f70bf..834dddd7b 100644
--- a/db/schema.rb
+++ b/db/schema.rb
@@ -10,7 +10,7 @@
 #
 # It's strongly recommended that you check this file into your version control system.
 
-ActiveRecord::Schema.define(version: 2019_09_01_040524) do
+ActiveRecord::Schema.define(version: 2019_09_04_222339) do
 
   # These are extensions that must be enabled in order to support this database
   enable_extension "plpgsql"
@@ -366,6 +366,17 @@ ActiveRecord::Schema.define(version: 2019_09_01_040524) do
     t.index ["account_id"], name: "index_lists_on_account_id"
   end
 
+  create_table "markers", force: :cascade do |t|
+    t.bigint "user_id"
+    t.string "timeline", default: "", null: false
+    t.bigint "last_read_id", default: 0, null: false
+    t.integer "lock_version", default: 0, null: false
+    t.datetime "created_at", null: false
+    t.datetime "updated_at", null: false
+    t.index ["user_id", "timeline"], name: "index_markers_on_user_id_and_timeline", unique: true
+    t.index ["user_id"], name: "index_markers_on_user_id"
+  end
+
   create_table "media_attachments", force: :cascade do |t|
     t.bigint "status_id"
     t.string "file_file_name"
@@ -791,6 +802,7 @@ ActiveRecord::Schema.define(version: 2019_09_01_040524) do
   add_foreign_key "list_accounts", "follows", on_delete: :cascade
   add_foreign_key "list_accounts", "lists", on_delete: :cascade
   add_foreign_key "lists", "accounts", on_delete: :cascade
+  add_foreign_key "markers", "users", on_delete: :cascade
   add_foreign_key "media_attachments", "accounts", name: "fk_96dd81e81b", on_delete: :nullify
   add_foreign_key "media_attachments", "scheduled_statuses", on_delete: :nullify
   add_foreign_key "media_attachments", "statuses", on_delete: :nullify
diff --git a/spec/controllers/api/v1/markers_controller_spec.rb b/spec/controllers/api/v1/markers_controller_spec.rb
new file mode 100644
index 000000000..556a75b9b
--- /dev/null
+++ b/spec/controllers/api/v1/markers_controller_spec.rb
@@ -0,0 +1,65 @@
+require 'rails_helper'
+
+RSpec.describe Api::V1::MarkersController, type: :controller do
+  render_views
+
+  let!(:user)  { Fabricate(:user, account: Fabricate(:account, username: 'alice')) }
+  let!(:token) { Fabricate(:accessible_access_token, resource_owner_id: user.id, scopes: 'read:statuses write:statuses') }
+
+  before { allow(controller).to receive(:doorkeeper_token) { token } }
+
+  describe 'GET #index' do
+    before do
+      Fabricate(:marker, timeline: 'home', last_read_id: 123, user: user)
+      Fabricate(:marker, timeline: 'notifications', last_read_id: 456, user: user)
+
+      get :index, params: { timeline: %w(home notifications) }
+    end
+
+    it 'returns http success' do
+      expect(response).to have_http_status(200)
+    end
+
+    it 'returns markers' do
+      json = body_as_json
+
+      expect(json.key?(:home)).to be true
+      expect(json[:home][:last_read_id]).to eq '123'
+      expect(json.key?(:notifications)).to be true
+      expect(json[:notifications][:last_read_id]).to eq '456'
+    end
+  end
+
+  describe 'POST #create' do
+    context 'when no marker exists' do
+      before do
+        post :create, params: { home: { last_read_id: '69420' } }
+      end
+
+      it 'returns http success' do
+        expect(response).to have_http_status(200)
+      end
+
+      it 'creates a marker' do
+        expect(user.markers.first.timeline).to eq 'home'
+        expect(user.markers.first.last_read_id).to eq 69420
+      end
+    end
+
+    context 'when a marker exists' do
+      before do
+        post :create, params: { home: { last_read_id: '69420' } }
+        post :create, params: { home: { last_read_id: '70120' } }
+      end
+
+      it 'returns http success' do
+        expect(response).to have_http_status(200)
+      end
+
+      it 'updates a marker' do
+        expect(user.markers.first.timeline).to eq 'home'
+        expect(user.markers.first.last_read_id).to eq 70120
+      end
+    end
+  end
+end
diff --git a/spec/fabricators/marker_fabricator.rb b/spec/fabricators/marker_fabricator.rb
new file mode 100644
index 000000000..0c94150e0
--- /dev/null
+++ b/spec/fabricators/marker_fabricator.rb
@@ -0,0 +1,6 @@
+Fabricator(:marker) do
+  user
+  timeline     'home'
+  last_read_id 0
+  lock_version 0
+end
diff --git a/spec/models/marker_spec.rb b/spec/models/marker_spec.rb
new file mode 100644
index 000000000..d716aa75c
--- /dev/null
+++ b/spec/models/marker_spec.rb
@@ -0,0 +1,5 @@
+require 'rails_helper'
+
+RSpec.describe Marker, type: :model do
+  pending "add some examples to (or delete) #{__FILE__}"
+end
-- 
cgit 


From a75009a65e1335047dff5488b4e67bdc03677590 Mon Sep 17 00:00:00 2001
From: Eugen Rochko <eugen@zeonfederated.com>
Date: Sun, 8 Sep 2019 19:17:57 +0200
Subject: Change half-life of trend decay (#11774)

---
 app/models/trending_tags.rb | 5 +++--
 1 file changed, 3 insertions(+), 2 deletions(-)

(limited to 'app/models')

diff --git a/app/models/trending_tags.rb b/app/models/trending_tags.rb
index e1b92b175..8cdade42d 100644
--- a/app/models/trending_tags.rb
+++ b/app/models/trending_tags.rb
@@ -7,8 +7,8 @@ class TrendingTags
   THRESHOLD            = 5
   LIMIT                = 10
   REVIEW_THRESHOLD     = 3
-  MAX_SCORE_COOLDOWN   = 3.days.freeze
-  MAX_SCORE_HALFLIFE   = 6.hours.freeze
+  MAX_SCORE_COOLDOWN   = 2.days.freeze
+  MAX_SCORE_HALFLIFE   = 2.hours.freeze
 
   class << self
     include Redisable
@@ -83,6 +83,7 @@ class TrendingTags
       # Trim older items
 
       redis.zremrangebyrank(KEY, 0, -(LIMIT + 1))
+      redis.zremrangebyscore(KEY, '(0.3', '-inf')
     end
 
     def get(limit, filtered: true)
-- 
cgit 


From 261e52268c05d2da4459a23e2898555dd5db5771 Mon Sep 17 00:00:00 2001
From: Eugen Rochko <eugen@zeonfederated.com>
Date: Mon, 9 Sep 2019 12:50:09 +0200
Subject: Add batch approve/reject for pending hashtags in admin UI (#11791)

---
 app/controllers/admin/tags_controller.rb   | 41 +++++++++++++++++++++++++++---
 app/javascript/styles/mastodon/tables.scss | 10 ++++++++
 app/models/form/tag_batch.rb               | 33 ++++++++++++++++++++++++
 app/views/admin/tags/_tag.html.haml        | 30 ++++++++++++----------
 app/views/admin/tags/index.html.haml       | 37 ++++++++++++++++++++++++++-
 config/locales/en.yml                      |  1 +
 config/routes.rb                           |  9 ++++++-
 7 files changed, 142 insertions(+), 19 deletions(-)
 create mode 100644 app/models/form/tag_batch.rb

(limited to 'app/models')

diff --git a/app/controllers/admin/tags_controller.rb b/app/controllers/admin/tags_controller.rb
index 8bd4e5f8b..376ebe44d 100644
--- a/app/controllers/admin/tags_controller.rb
+++ b/app/controllers/admin/tags_controller.rb
@@ -3,12 +3,33 @@
 module Admin
   class TagsController < BaseController
     before_action :set_tags, only: :index
-    before_action :set_tag, except: :index
-    before_action :set_usage_by_domain, except: :index
-    before_action :set_counters, except: :index
+    before_action :set_tag, except: [:index, :batch, :approve_all, :reject_all]
+    before_action :set_usage_by_domain, except: [:index, :batch, :approve_all, :reject_all]
+    before_action :set_counters, except: [:index, :batch, :approve_all, :reject_all]
 
     def index
       authorize :tag, :index?
+
+      @form = Form::TagBatch.new
+    end
+
+    def batch
+      @form = Form::TagBatch.new(form_tag_batch_params.merge(current_account: current_account, action: action_from_button))
+      @form.save
+    rescue ActionController::ParameterMissing
+      flash[:alert] = I18n.t('admin.accounts.no_account_selected')
+    ensure
+      redirect_to admin_tags_path(filter_params)
+    end
+
+    def approve_all
+      Form::TagBatch.new(current_account: current_account, tag_ids: Tag.pending_review.pluck(:id), action: 'approve').save
+      redirect_to admin_tags_path(filter_params)
+    end
+
+    def reject_all
+      Form::TagBatch.new(current_account: current_account, tag_ids: Tag.pending_review.pluck(:id), action: 'reject').save
+      redirect_to admin_tags_path(filter_params)
     end
 
     def show
@@ -61,7 +82,7 @@ module Admin
     end
 
     def filter_params
-      params.slice(:context, :review).permit(:context, :review)
+      params.slice(:context, :review, :page).permit(:context, :review, :page)
     end
 
     def tag_params
@@ -75,5 +96,17 @@ module Admin
         date.to_time(:utc).beginning_of_day.to_i
       end
     end
+
+    def form_tag_batch_params
+      params.require(:form_tag_batch).permit(:action, tag_ids: [])
+    end
+
+    def action_from_button
+      if params[:approve]
+        'approve'
+      elsif params[:reject]
+        'reject'
+      end
+    end
   end
 end
diff --git a/app/javascript/styles/mastodon/tables.scss b/app/javascript/styles/mastodon/tables.scss
index fe6beba5d..2aef099e6 100644
--- a/app/javascript/styles/mastodon/tables.scss
+++ b/app/javascript/styles/mastodon/tables.scss
@@ -211,6 +211,16 @@ a.table-action-link {
         padding: 0;
       }
     }
+
+    .directory__tag {
+      margin: 0;
+      width: 100%;
+
+      a {
+        background: transparent;
+        border-radius: 0;
+      }
+    }
   }
 
   .status__content {
diff --git a/app/models/form/tag_batch.rb b/app/models/form/tag_batch.rb
new file mode 100644
index 000000000..fd517a1a6
--- /dev/null
+++ b/app/models/form/tag_batch.rb
@@ -0,0 +1,33 @@
+# frozen_string_literal: true
+
+class Form::TagBatch
+  include ActiveModel::Model
+  include Authorization
+
+  attr_accessor :tag_ids, :action, :current_account
+
+  def save
+    case action
+    when 'approve'
+      approve!
+    when 'reject'
+      reject!
+    end
+  end
+
+  private
+
+  def tags
+    Tag.where(id: tag_ids)
+  end
+
+  def approve!
+    tags.each { |tag| authorize(tag, :update?) }
+    tags.update_all(trendable: true, reviewed_at: Time.now.utc)
+  end
+
+  def reject!
+    tags.each { |tag| authorize(tag, :update?) }
+    tags.update_all(trendable: false, reviewed_at: Time.now.utc)
+  end
+end
diff --git a/app/views/admin/tags/_tag.html.haml b/app/views/admin/tags/_tag.html.haml
index 91af8e492..670f3bc05 100644
--- a/app/views/admin/tags/_tag.html.haml
+++ b/app/views/admin/tags/_tag.html.haml
@@ -1,16 +1,20 @@
-.directory__tag
-  = link_to admin_tag_path(tag.id) do
-    %h4
-      = fa_icon 'hashtag'
-      = tag.name
+.batch-table__row
+  %label.batch-table__row__select.batch-table__row__select--aligned.batch-checkbox
+    = f.check_box :tag_ids, { multiple: true, include_hidden: false }, tag.id
 
-      %small
-        = t('admin.tags.in_directory', count: tag.accounts_count)
-        &bull;
-        = t('admin.tags.unique_uses_today', count: tag.history.first[:accounts])
+  .directory__tag
+    = link_to admin_tag_path(tag.id) do
+      %h4
+        = fa_icon 'hashtag'
+        = tag.name
 
-        - if tag.trending?
-          = fa_icon 'fire fw'
-          = t('admin.tags.trending_right_now')
+        %small
+          = t('admin.tags.in_directory', count: tag.accounts_count)
+          &bull;
+          = t('admin.tags.unique_uses_today', count: tag.history.first[:accounts])
 
-    .trends__item__current= number_to_human tag.history.first[:uses], strip_insignificant_zeros: true
+          - if tag.trending?
+            = fa_icon 'fire fw'
+            = t('admin.tags.trending_right_now')
+
+      .trends__item__current= number_to_human tag.history.first[:uses], strip_insignificant_zeros: true
diff --git a/app/views/admin/tags/index.html.haml b/app/views/admin/tags/index.html.haml
index d994955ef..324d13d3e 100644
--- a/app/views/admin/tags/index.html.haml
+++ b/app/views/admin/tags/index.html.haml
@@ -1,6 +1,9 @@
 - content_for :page_title do
   = t('admin.tags.title')
 
+- content_for :header_tags do
+  = javascript_pack_tag 'admin', integrity: true, async: true, crossorigin: 'anonymous'
+
 .filters
   .filter-subset
     %strong= t('admin.tags.context')
@@ -18,5 +21,37 @@
 
 %hr.spacer/
 
-= render @tags
+= form_for(@form, url: batch_admin_tags_path) do |f|
+  = hidden_field_tag :page, params[:page] || 1
+  = hidden_field_tag :context, params[:context]
+  = hidden_field_tag :review, params[:review]
+
+  .batch-table
+    .batch-table__toolbar
+      %label.batch-table__toolbar__select.batch-checkbox-all
+        = check_box_tag :batch_checkbox_all, nil, false
+      .batch-table__toolbar__actions
+        - if params[:review] == 'pending_review'
+          = f.button safe_join([fa_icon('check'), t('admin.accounts.approve')]), name: :approve, class: 'table-action-link', type: :submit, data: { confirm: t('admin.reports.are_you_sure') }
+
+          = f.button safe_join([fa_icon('times'), t('admin.accounts.reject')]), name: :reject, class: 'table-action-link', type: :submit, data: { confirm: t('admin.reports.are_you_sure') }
+        - else
+          %span.neutral-hint= t('generic.no_batch_actions_available')
+
+    .batch-table__body
+      - if @tags.empty?
+        = nothing_here 'nothing-here--under-tabs'
+      - else
+        = render partial: 'tag', collection: @tags, locals: { f: f }
+
 = paginate @tags
+
+- if params[:review] == 'pending_review'
+  %hr.spacer/
+
+  %div{ style: 'overflow: hidden' }
+    %div{ style: 'float: right' }
+      = link_to t('admin.accounts.reject_all'), reject_all_admin_tags_path, method: :post, data: { confirm: t('admin.accounts.are_you_sure') }, class: 'button button--destructive'
+
+    %div
+      = link_to t('admin.accounts.approve_all'), approve_all_admin_tags_path, method: :post, data: { confirm: t('admin.accounts.are_you_sure') }, class: 'button'
diff --git a/config/locales/en.yml b/config/locales/en.yml
index 687f5f2a0..42d8e0eb8 100644
--- a/config/locales/en.yml
+++ b/config/locales/en.yml
@@ -727,6 +727,7 @@ en:
     all: All
     changes_saved_msg: Changes successfully saved!
     copy: Copy
+    no_batch_actions_available: No batch actions available on this page
     order_by: Order by
     save_changes: Save changes
     validation_errors:
diff --git a/config/routes.rb b/config/routes.rb
index 1ebf9e066..534e68814 100644
--- a/config/routes.rb
+++ b/config/routes.rb
@@ -251,7 +251,14 @@ Rails.application.routes.draw do
     end
 
     resources :account_moderation_notes, only: [:create, :destroy]
-    resources :tags, only: [:index, :show, :update]
+
+    resources :tags, only: [:index, :show, :update] do
+      collection do
+        post :approve_all
+        post :reject_all
+        post :batch
+      end
+    end
   end
 
   get '/admin', to: redirect('/admin/dashboard', status: 302)
-- 
cgit 


From 1110ea1a9162d5488e1ed5dbccd0803618e713f8 Mon Sep 17 00:00:00 2001
From: Eugen Rochko <eugen@zeonfederated.com>
Date: Mon, 9 Sep 2019 22:44:17 +0200
Subject: Add batch actions and categories to admin UI for custom emojis
 (#11793)

---
 app/controllers/admin/custom_emojis_controller.rb  | 102 +++++++-------------
 app/javascript/styles/mastodon/tables.scss         |  41 ++++++++
 app/models/custom_emoji.rb                         |   6 ++
 app/models/custom_emoji_category.rb                |   2 +
 app/models/custom_emoji_filter.rb                  |   8 +-
 app/models/form/custom_emoji_batch.rb              | 106 +++++++++++++++++++++
 .../admin/custom_emojis/_custom_emoji.html.haml    |  55 ++++++-----
 app/views/admin/custom_emojis/index.html.haml      |  66 ++++++++++---
 config/locales/en.yml                              |   3 +
 config/routes.rb                                   |   8 +-
 .../admin/custom_emojis_controller_spec.rb         |  60 ------------
 11 files changed, 281 insertions(+), 176 deletions(-)
 create mode 100644 app/models/form/custom_emoji_batch.rb

(limited to 'app/models')

diff --git a/app/controllers/admin/custom_emojis_controller.rb b/app/controllers/admin/custom_emojis_controller.rb
index f77699166..2af90f051 100644
--- a/app/controllers/admin/custom_emojis_controller.rb
+++ b/app/controllers/admin/custom_emojis_controller.rb
@@ -2,19 +2,20 @@
 
 module Admin
   class CustomEmojisController < BaseController
-    before_action :set_custom_emoji, except: [:index, :new, :create]
-    before_action :set_filter_params
-
     include ObfuscateFilename
+
     obfuscate_filename [:custom_emoji, :image]
 
     def index
       authorize :custom_emoji, :index?
+
       @custom_emojis = filtered_custom_emojis.eager_load(:local_counterpart).page(params[:page])
+      @form          = Form::CustomEmojiBatch.new
     end
 
     def new
       authorize :custom_emoji, :create?
+
       @custom_emoji = CustomEmoji.new
     end
 
@@ -31,69 +32,17 @@ module Admin
       end
     end
 
-    def update
-      authorize @custom_emoji, :update?
-
-      if @custom_emoji.update(resource_params)
-        log_action :update, @custom_emoji
-        flash[:notice] = I18n.t('admin.custom_emojis.updated_msg')
-      else
-        flash[:alert] =  I18n.t('admin.custom_emojis.update_failed_msg')
-      end
-      redirect_to admin_custom_emojis_path(page: params[:page], **@filter_params)
-    end
-
-    def destroy
-      authorize @custom_emoji, :destroy?
-      @custom_emoji.destroy!
-      log_action :destroy, @custom_emoji
-      flash[:notice] = I18n.t('admin.custom_emojis.destroyed_msg')
-      redirect_to admin_custom_emojis_path(page: params[:page], **@filter_params)
-    end
-
-    def copy
-      authorize @custom_emoji, :copy?
-
-      emoji = CustomEmoji.find_or_initialize_by(domain: nil,
-                                                shortcode: @custom_emoji.shortcode)
-      emoji.image = @custom_emoji.image
-
-      if emoji.save
-        log_action :create, emoji
-        flash[:notice] = I18n.t('admin.custom_emojis.copied_msg')
-      else
-        flash[:alert] = I18n.t('admin.custom_emojis.copy_failed_msg')
-      end
-
-      redirect_to admin_custom_emojis_path(page: params[:page], **@filter_params)
-    end
-
-    def enable
-      authorize @custom_emoji, :enable?
-      @custom_emoji.update!(disabled: false)
-      log_action :enable, @custom_emoji
-      flash[:notice] = I18n.t('admin.custom_emojis.enabled_msg')
-      redirect_to admin_custom_emojis_path(page: params[:page], **@filter_params)
-    end
-
-    def disable
-      authorize @custom_emoji, :disable?
-      @custom_emoji.update!(disabled: true)
-      log_action :disable, @custom_emoji
-      flash[:notice] = I18n.t('admin.custom_emojis.disabled_msg')
-      redirect_to admin_custom_emojis_path(page: params[:page], **@filter_params)
+    def batch
+      @form = Form::CustomEmojiBatch.new(form_custom_emoji_batch_params.merge(current_account: current_account, action: action_from_button))
+      @form.save
+    rescue ActionController::ParameterMissing
+      flash[:alert] = I18n.t('admin.accounts.no_account_selected')
+    ensure
+      redirect_to admin_custom_emojis_path(filter_params)
     end
 
     private
 
-    def set_custom_emoji
-      @custom_emoji = CustomEmoji.find(params[:id])
-    end
-
-    def set_filter_params
-      @filter_params = filter_params.to_hash.symbolize_keys
-    end
-
     def resource_params
       params.require(:custom_emoji).permit(:shortcode, :image, :visible_in_picker)
     end
@@ -103,12 +52,29 @@ module Admin
     end
 
     def filter_params
-      params.permit(
-        :local,
-        :remote,
-        :by_domain,
-        :shortcode
-      )
+      params.slice(:local, :remote, :by_domain, :shortcode, :page).permit(:local, :remote, :by_domain, :shortcode, :page)
+    end
+
+    def action_from_button
+      if params[:update]
+        'update'
+      elsif params[:list]
+        'list'
+      elsif params[:unlist]
+        'unlist'
+      elsif params[:enable]
+        'enable'
+      elsif params[:disable]
+        'disable'
+      elsif params[:copy]
+        'copy'
+      elsif params[:delete]
+        'delete'
+      end
+    end
+
+    def form_custom_emoji_batch_params
+      params.require(:form_custom_emoji_batch).permit(:action, :category_id, :category_name, custom_emoji_ids: [])
     end
   end
 end
diff --git a/app/javascript/styles/mastodon/tables.scss b/app/javascript/styles/mastodon/tables.scss
index 2aef099e6..d6403986f 100644
--- a/app/javascript/styles/mastodon/tables.scss
+++ b/app/javascript/styles/mastodon/tables.scss
@@ -180,6 +180,18 @@ a.table-action-link {
     }
   }
 
+  &__form {
+    padding: 16px;
+    border: 1px solid darken($ui-base-color, 8%);
+    border-top: 0;
+    background: $ui-base-color;
+
+    .fields-row {
+      padding-top: 0;
+      margin-bottom: 0;
+    }
+  }
+
   &__row {
     border: 1px solid darken($ui-base-color, 8%);
     border-top: 0;
@@ -210,6 +222,35 @@ a.table-action-link {
       &--unpadded {
         padding: 0;
       }
+
+      &--with-image {
+        display: flex;
+        align-items: center;
+      }
+
+      &__image {
+        flex: 0 0 auto;
+        display: flex;
+        justify-content: center;
+        align-items: center;
+        margin-right: 10px;
+
+        .emojione {
+          width: 32px;
+          height: 32px;
+        }
+      }
+
+      &__text {
+        flex: 1 1 auto;
+      }
+
+      &__extra {
+        flex: 0 0 auto;
+        text-align: right;
+        color: $darker-text-color;
+        font-weight: 500;
+      }
     }
 
     .directory__tag {
diff --git a/app/models/custom_emoji.rb b/app/models/custom_emoji.rb
index b21ad9042..0a4201a14 100644
--- a/app/models/custom_emoji.rb
+++ b/app/models/custom_emoji.rb
@@ -59,6 +59,12 @@ class CustomEmoji < ApplicationRecord
     :emoji
   end
 
+  def copy!
+    copy = self.class.find_or_initialize_by(domain: nil, shortcode: shortcode)
+    copy.image = image
+    copy.save!
+  end
+
   class << self
     def from_text(text, domain)
       return [] if text.blank?
diff --git a/app/models/custom_emoji_category.rb b/app/models/custom_emoji_category.rb
index 7d8c0ee2d..3c87f2b2e 100644
--- a/app/models/custom_emoji_category.rb
+++ b/app/models/custom_emoji_category.rb
@@ -12,4 +12,6 @@
 
 class CustomEmojiCategory < ApplicationRecord
   has_many :emojis, class_name: 'CustomEmoji', foreign_key: 'category_id', inverse_of: :category
+
+  validates :name, presence: true, uniqueness: true
 end
diff --git a/app/models/custom_emoji_filter.rb b/app/models/custom_emoji_filter.rb
index 7649055d2..15b8da1d1 100644
--- a/app/models/custom_emoji_filter.rb
+++ b/app/models/custom_emoji_filter.rb
@@ -11,6 +11,8 @@ class CustomEmojiFilter
     scope = CustomEmoji.alphabetic
 
     params.each do |key, value|
+      next if key.to_s == 'page'
+
       scope.merge!(scope_for(key, value)) if value.present?
     end
 
@@ -22,13 +24,13 @@ class CustomEmojiFilter
   def scope_for(key, value)
     case key.to_s
     when 'local'
-      CustomEmoji.local
+      CustomEmoji.local.left_joins(:category).reorder(Arel.sql('custom_emoji_categories.name ASC NULLS FIRST, custom_emojis.shortcode ASC'))
     when 'remote'
       CustomEmoji.remote
     when 'by_domain'
-      CustomEmoji.where(domain: value.downcase)
+      CustomEmoji.where(domain: value.strip.downcase)
     when 'shortcode'
-      CustomEmoji.search(value)
+      CustomEmoji.search(value.strip)
     else
       raise "Unknown filter: #{key}"
     end
diff --git a/app/models/form/custom_emoji_batch.rb b/app/models/form/custom_emoji_batch.rb
new file mode 100644
index 000000000..076e8c9e3
--- /dev/null
+++ b/app/models/form/custom_emoji_batch.rb
@@ -0,0 +1,106 @@
+# frozen_string_literal: true
+
+class Form::CustomEmojiBatch
+  include ActiveModel::Model
+  include Authorization
+  include AccountableConcern
+
+  attr_accessor :custom_emoji_ids, :action, :current_account,
+                :category_id, :category_name, :visible_in_picker
+
+  def save
+    case action
+    when 'update'
+      update!
+    when 'list'
+      list!
+    when 'unlist'
+      unlist!
+    when 'enable'
+      enable!
+    when 'disable'
+      disable!
+    when 'copy'
+      copy!
+    when 'delete'
+      delete!
+    end
+  end
+
+  private
+
+  def custom_emojis
+    CustomEmoji.where(id: custom_emoji_ids)
+  end
+
+  def update!
+    custom_emojis.each { |custom_emoji| authorize(custom_emoji, :update?) }
+
+    category = begin
+      if category_id.present?
+        CustomEmojiCategory.find(category_id)
+      elsif category_name.present?
+        CustomEmojiCategory.create!(name: category_name)
+      end
+    end
+
+    custom_emojis.each do |custom_emoji|
+      custom_emoji.update(category_id: category&.id)
+      log_action :update, custom_emoji
+    end
+  end
+
+  def list!
+    custom_emojis.each { |custom_emoji| authorize(custom_emoji, :update?) }
+
+    custom_emojis.each do |custom_emoji|
+      custom_emoji.update(visible_in_picker: true)
+      log_action :update, custom_emoji
+    end
+  end
+
+  def unlist!
+    custom_emojis.each { |custom_emoji| authorize(custom_emoji, :update?) }
+
+    custom_emojis.each do |custom_emoji|
+      custom_emoji.update(visible_in_picker: false)
+      log_action :update, custom_emoji
+    end
+  end
+
+  def enable!
+    custom_emojis.each { |custom_emoji| authorize(custom_emoji, :enable?) }
+
+    custom_emojis.each do |custom_emoji|
+      custom_emoji.update(disabled: false)
+      log_action :enable, custom_emoji
+    end
+  end
+
+  def disable!
+    custom_emojis.each { |custom_emoji| authorize(custom_emoji, :disable?) }
+
+    custom_emojis.each do |custom_emoji|
+      custom_emoji.update(disabled: true)
+      log_action :disable, custom_emoji
+    end
+  end
+
+  def copy!
+    custom_emojis.each { |custom_emoji| authorize(custom_emoji, :copy?) }
+
+    custom_emojis.each do |custom_emoji|
+      copied_custom_emoji = custom_emoji.copy!
+      log_action :create, copied_custom_emoji
+    end
+  end
+
+  def delete!
+    custom_emojis.each { |custom_emoji| authorize(custom_emoji, :destroy?) }
+
+    custom_emojis.each do |custom_emoji|
+      custom_emoji.destroy
+      log_action :destroy, custom_emoji
+    end
+  end
+end
diff --git a/app/views/admin/custom_emojis/_custom_emoji.html.haml b/app/views/admin/custom_emojis/_custom_emoji.html.haml
index fbaa9a174..9e06a3b42 100644
--- a/app/views/admin/custom_emojis/_custom_emoji.html.haml
+++ b/app/views/admin/custom_emojis/_custom_emoji.html.haml
@@ -1,28 +1,31 @@
-%tr
-  %td
-    = custom_emoji_tag(custom_emoji)
-  %td
-    %samp= ":#{custom_emoji.shortcode}:"
-  %td
-    - if custom_emoji.local?
-      = t('admin.accounts.location.local')
-    - else
-      = link_to custom_emoji.domain, admin_custom_emojis_path(by_domain: custom_emoji.domain)
-  %td
-    - if custom_emoji.local?
-      - if custom_emoji.visible_in_picker
-        = table_link_to 'eye', t('admin.custom_emojis.listed'), admin_custom_emoji_path(custom_emoji, custom_emoji: { visible_in_picker: false }, page: params[:page], **@filter_params), method: :patch
+.batch-table__row
+  %label.batch-table__row__select.batch-table__row__select--aligned.batch-checkbox
+    = f.check_box :custom_emoji_ids, { multiple: true, include_hidden: false }, custom_emoji.id
+  .batch-table__row__content.batch-table__row__content--with-image
+    .batch-table__row__content__image
+      = custom_emoji_tag(custom_emoji)
+
+    .batch-table__row__content__text
+      %samp= ":#{custom_emoji.shortcode}:"
+
+      - if custom_emoji.local?
+        %span.account-role.bot= custom_emoji.category&.name || t('admin.custom_emojis.uncategorized')
+
+    .batch-table__row__content__extra
+      - if custom_emoji.local?
+        = t('admin.accounts.location.local')
       - else
-        = table_link_to 'eye-slash', t('admin.custom_emojis.unlisted'), admin_custom_emoji_path(custom_emoji, custom_emoji: { visible_in_picker: true }, page: params[:page], **@filter_params), method: :patch
-    - else
-      - if custom_emoji.local_counterpart.present?
-        = link_to safe_join([custom_emoji_tag(custom_emoji.local_counterpart), t('admin.custom_emojis.overwrite')]), copy_admin_custom_emoji_path(custom_emoji, page: params[:page], **@filter_params), method: :post, class: 'table-action-link'
+        = custom_emoji.domain
+
+      %br/
+
+      - if custom_emoji.disabled?
+        = t('admin.custom_emojis.disabled')
       - else
-        = table_link_to 'copy', t('admin.custom_emojis.copy'), copy_admin_custom_emoji_path(custom_emoji, page: params[:page], **@filter_params), method: :post
-  %td
-    - if custom_emoji.disabled?
-      = table_link_to 'power-off', t('admin.custom_emojis.enable'), enable_admin_custom_emoji_path(custom_emoji, page: params[:page], **@filter_params), method: :post, data: { confirm: t('admin.accounts.are_you_sure') }
-    - else
-      = table_link_to 'power-off', t('admin.custom_emojis.disable'), disable_admin_custom_emoji_path(custom_emoji, page: params[:page], **@filter_params), method: :post, data: { confirm: t('admin.accounts.are_you_sure') }
-  %td
-    = table_link_to 'times', t('admin.custom_emojis.delete'), admin_custom_emoji_path(custom_emoji, page: params[:page], **@filter_params), method: :delete, data: { confirm: t('admin.accounts.are_you_sure') }
+        = t('admin.custom_emojis.enabled')
+      - if custom_emoji.local?
+        &bull;
+        - if custom_emoji.visible_in_picker?
+          = t('admin.custom_emojis.listed')
+        - else
+          = t('admin.custom_emojis.unlisted')
diff --git a/app/views/admin/custom_emojis/index.html.haml b/app/views/admin/custom_emojis/index.html.haml
index 3a119276c..7320ce1bb 100644
--- a/app/views/admin/custom_emojis/index.html.haml
+++ b/app/views/admin/custom_emojis/index.html.haml
@@ -1,6 +1,9 @@
 - content_for :page_title do
   = t('admin.custom_emojis.title')
 
+- content_for :header_tags do
+  = javascript_pack_tag 'admin', integrity: true, async: true, crossorigin: 'anonymous'
+
 .filters
   .filter-subset
     %strong= t('admin.accounts.location.title')
@@ -20,8 +23,7 @@
 = form_tag admin_custom_emojis_url, method: 'GET', class: 'simple_form' do
   .fields-group
     - Admin::FilterHelper::CUSTOM_EMOJI_FILTERS.each do |key|
-      - if params[key].present?
-        = hidden_field_tag key, params[key]
+      = hidden_field_tag key, params[key] if params[key].present?
 
     - %i(shortcode by_domain).each do |key|
       .input.string.optional
@@ -31,18 +33,54 @@
       %button= t('admin.accounts.search')
       = link_to t('admin.accounts.reset'), admin_custom_emojis_path, class: 'button negative'
 
-.table-wrapper
-  %table.table
-    %thead
-      %tr
-        %th= t('admin.custom_emojis.emoji')
-        %th= t('admin.custom_emojis.shortcode')
-        %th= t('admin.accounts.domain')
-        %th
-        %th
-        %th
-    %tbody
-      = render @custom_emojis
+= form_for(@form, url: batch_admin_custom_emojis_path) do |f|
+  = hidden_field_tag :page, params[:page] || 1
+
+  - Admin::FilterHelper::CUSTOM_EMOJI_FILTERS.each do |key|
+    = hidden_field_tag key, params[key] if params[key].present?
+
+  .batch-table
+    .batch-table__toolbar
+      %label.batch-table__toolbar__select.batch-checkbox-all
+        = check_box_tag :batch_checkbox_all, nil, false
+      .batch-table__toolbar__actions
+        - if params[:local] == '1'
+          = f.button safe_join([fa_icon('save'), t('generic.save_changes')]), name: :update, class: 'table-action-link', type: :submit, data: { confirm: t('admin.reports.are_you_sure') }
+
+          = f.button safe_join([fa_icon('eye'), t('admin.custom_emojis.list')]), name: :list, class: 'table-action-link', type: :submit, data: { confirm: t('admin.reports.are_you_sure') }
+
+          = f.button safe_join([fa_icon('eye-slash'), t('admin.custom_emojis.unlist')]), name: :unlist, class: 'table-action-link', type: :submit, data: { confirm: t('admin.reports.are_you_sure') }
+
+        = f.button safe_join([fa_icon('power-off'), t('admin.custom_emojis.enable')]), name: :enable, class: 'table-action-link', type: :submit, data: { confirm: t('admin.reports.are_you_sure') }
+
+        = f.button safe_join([fa_icon('power-off'), t('admin.custom_emojis.disable')]), name: :disable, class: 'table-action-link', type: :submit, data: { confirm: t('admin.reports.are_you_sure') }
+
+        = f.button safe_join([fa_icon('times'), t('admin.custom_emojis.delete')]), name: :delete, class: 'table-action-link', type: :submit, data: { confirm: t('admin.reports.are_you_sure') }
+
+        - unless params[:local] == '1'
+          = f.button safe_join([fa_icon('copy'), t('admin.custom_emojis.copy')]), name: :copy, class: 'table-action-link', type: :submit, data: { confirm: t('admin.reports.are_you_sure') }
+
+    - if params[:local] == '1'
+      .batch-table__form.simple_form
+        .fields-row
+          .fields-group.fields-row__column.fields-row__column-6
+            .input.select.optional
+              .label_input
+                = f.select :category_id, options_from_collection_for_select(CustomEmojiCategory.all, 'id', 'name'), prompt: t('admin.custom_emojis.assign_category'), class: 'select optional', 'aria-label': t('admin.custom_emojis.assign_category')
+
+          .fields-group.fields-row__column.fields-row__column-6
+            .input.string.optional
+              .label_input
+                = f.text_field :category_name, class: 'string optional', placeholder: t('admin.custom_emojis.create_new_category'), 'aria-label': t('admin.custom_emojis.create_new_category')
+
+    .batch-table__body
+      - if @custom_emojis.empty?
+        = nothing_here 'nothing-here--under-tabs'
+      - else
+        = render partial: 'custom_emoji', collection: @custom_emojis, locals: { f: f }
 
 = paginate @custom_emojis
+
+%hr.spacer/
+
 = link_to t('admin.custom_emojis.upload'), new_admin_custom_emoji_path, class: 'button'
diff --git a/config/locales/en.yml b/config/locales/en.yml
index 42d8e0eb8..52cb4a269 100644
--- a/config/locales/en.yml
+++ b/config/locales/en.yml
@@ -225,10 +225,12 @@ en:
       deleted_status: "(deleted status)"
       title: Audit log
     custom_emojis:
+      assign_category: Assign category
       by_domain: Domain
       copied_msg: Successfully created local copy of the emoji
       copy: Copy
       copy_failed_msg: Could not make a local copy of that emoji
+      create_new_category: Create new category
       created_msg: Emoji successfully created!
       delete: Delete
       destroyed_msg: Emojo successfully destroyed!
@@ -245,6 +247,7 @@ en:
       shortcode: Shortcode
       shortcode_hint: At least 2 characters, only alphanumeric characters and underscores
       title: Custom emojis
+      uncategorized: Uncategorized
       unlisted: Unlisted
       update_failed_msg: Could not update that emoji
       updated_msg: Emoji successfully updated!
diff --git a/config/routes.rb b/config/routes.rb
index 534e68814..d22a9e56a 100644
--- a/config/routes.rb
+++ b/config/routes.rb
@@ -242,11 +242,9 @@ Rails.application.routes.draw do
       resource :two_factor_authentication, only: [:destroy]
     end
 
-    resources :custom_emojis, only: [:index, :new, :create, :update, :destroy] do
-      member do
-        post :copy
-        post :enable
-        post :disable
+    resources :custom_emojis, only: [:index, :new, :create] do
+      collection do
+        post :batch
       end
     end
 
diff --git a/spec/controllers/admin/custom_emojis_controller_spec.rb b/spec/controllers/admin/custom_emojis_controller_spec.rb
index b7e2894e9..a8d96948c 100644
--- a/spec/controllers/admin/custom_emojis_controller_spec.rb
+++ b/spec/controllers/admin/custom_emojis_controller_spec.rb
@@ -52,64 +52,4 @@ describe Admin::CustomEmojisController do
       end
     end
   end
-
-  describe 'PUT #update' do
-    let(:custom_emoji) { Fabricate(:custom_emoji, shortcode: 'test') }
-    let(:image) { fixture_file_upload(Rails.root.join('spec', 'fixtures', 'files', 'emojo.png'), 'image/png') }
-
-    before do
-      put :update, params: { id: custom_emoji.id, custom_emoji: params }
-    end
-
-    context 'when parameter is valid' do
-      let(:params) { { shortcode: 'updated', image: image } }
-
-      it 'succeeds in updating custom emoji' do
-        expect(flash[:notice]).to eq I18n.t('admin.custom_emojis.updated_msg')
-        expect(custom_emoji.reload).to have_attributes(shortcode: 'updated')
-      end
-    end
-
-    context 'when parameter is invalid' do
-      let(:params) { { shortcode: 'u', image: image } }
-
-      it 'fails to update custom emoji' do
-        expect(flash[:alert]).to eq I18n.t('admin.custom_emojis.update_failed_msg')
-        expect(custom_emoji.reload).to have_attributes(shortcode: 'test')
-      end
-    end
-  end
-
-  describe 'POST #copy' do
-    subject { post :copy, params: { id: custom_emoji.id } }
-
-    let(:custom_emoji) { Fabricate(:custom_emoji, shortcode: 'test') }
-
-    it 'copies custom emoji' do
-      expect { subject }.to change { CustomEmoji.where(shortcode: 'test').count }.by(1)
-      expect(flash[:notice]).to eq I18n.t('admin.custom_emojis.copied_msg')
-    end
-  end
-
-  describe 'POST #enable' do
-    let(:custom_emoji) { Fabricate(:custom_emoji, shortcode: 'test', disabled: true) }
-
-    before { post :enable, params: { id: custom_emoji.id } }
-
-    it 'enables custom emoji' do
-      expect(response).to redirect_to admin_custom_emojis_path
-      expect(custom_emoji.reload).to have_attributes(disabled: false)
-    end
-  end
-
-  describe 'POST #disable' do
-    let(:custom_emoji) { Fabricate(:custom_emoji, shortcode: 'test', disabled: false) }
-
-    before { post :disable, params: { id: custom_emoji.id } }
-
-    it 'enables custom emoji' do
-      expect(response).to redirect_to admin_custom_emojis_path
-      expect(custom_emoji.reload).to have_attributes(disabled: true)
-    end
-  end
 end
-- 
cgit 


From 86748148256b504c0411119628435b1445959309 Mon Sep 17 00:00:00 2001
From: Eugen Rochko <eugen@zeonfederated.com>
Date: Tue, 10 Sep 2019 13:48:48 +0200
Subject: Change tootctl to use inline parallelization instead of Sidekiq
 (#11776)

- Remove --background option
- Add --concurrency(=5) option
- Add progress bars
---
 Gemfile                                            |   1 +
 Gemfile.lock                                       |   1 +
 app/models/media_attachment.rb                     |   1 +
 app/models/preview_card.rb                         |   2 +
 app/workers/maintenance/destroy_media_worker.rb    |  14 --
 .../maintenance/redownload_account_media_worker.rb |  16 --
 app/workers/maintenance/uncache_media_worker.rb    |  18 --
 app/workers/maintenance/uncache_preview_worker.rb  |  18 --
 lib/mastodon/accounts_cli.rb                       | 192 +++++++++------------
 lib/mastodon/cache_cli.rb                          |  16 +-
 lib/mastodon/cli_helper.rb                         |  49 ++++++
 lib/mastodon/domains_cli.rb                        |  27 ++-
 lib/mastodon/feeds_cli.rb                          |  42 +----
 lib/mastodon/media_cli.rb                          |  57 ++----
 lib/mastodon/preview_cards_cli.rb                  |  83 +++------
 15 files changed, 200 insertions(+), 337 deletions(-)
 delete mode 100644 app/workers/maintenance/destroy_media_worker.rb
 delete mode 100644 app/workers/maintenance/redownload_account_media_worker.rb
 delete mode 100644 app/workers/maintenance/uncache_media_worker.rb
 delete mode 100644 app/workers/maintenance/uncache_preview_worker.rb

(limited to 'app/models')

diff --git a/Gemfile b/Gemfile
index 73edb2a6a..af0e8e2fc 100644
--- a/Gemfile
+++ b/Gemfile
@@ -77,6 +77,7 @@ gem 'rails-settings-cached', '~> 0.6'
 gem 'redis', '~> 4.1', require: ['redis', 'redis/connection/hiredis']
 gem 'mario-redis-lock', '~> 1.2', require: 'redis_lock'
 gem 'rqrcode', '~> 0.10'
+gem 'ruby-progressbar', '~> 1.10'
 gem 'sanitize', '~> 5.1'
 gem 'sidekiq', '~> 5.2'
 gem 'sidekiq-scheduler', '~> 3.0'
diff --git a/Gemfile.lock b/Gemfile.lock
index 125cca4ba..6e931c611 100644
--- a/Gemfile.lock
+++ b/Gemfile.lock
@@ -769,6 +769,7 @@ DEPENDENCIES
   rspec-sidekiq (~> 3.0)
   rubocop (~> 0.74)
   rubocop-rails (~> 2.3)
+  ruby-progressbar (~> 1.10)
   sanitize (~> 5.1)
   sidekiq (~> 5.2)
   sidekiq-bulk (~> 0.2.0)
diff --git a/app/models/media_attachment.rb b/app/models/media_attachment.rb
index 44f8e6be6..b58025015 100644
--- a/app/models/media_attachment.rb
+++ b/app/models/media_attachment.rb
@@ -129,6 +129,7 @@ class MediaAttachment < ApplicationRecord
   scope :unattached, -> { where(status_id: nil, scheduled_status_id: nil) }
   scope :local,      -> { where(remote_url: '') }
   scope :remote,     -> { where.not(remote_url: '') }
+  scope :cached,     -> { remote.where.not(file_file_name: nil) }
 
   default_scope { order(id: :asc) }
 
diff --git a/app/models/preview_card.rb b/app/models/preview_card.rb
index a792b352b..9d6c1938a 100644
--- a/app/models/preview_card.rb
+++ b/app/models/preview_card.rb
@@ -43,6 +43,8 @@ class PreviewCard < ApplicationRecord
   validates_attachment_size :image, less_than: LIMIT
   remotable_attachment :image, LIMIT
 
+  scope :cached, -> { where.not(image_file_name: [nil, '']) }
+
   before_save :extract_dimensions, if: :link?
 
   def save_with_optional_image!
diff --git a/app/workers/maintenance/destroy_media_worker.rb b/app/workers/maintenance/destroy_media_worker.rb
deleted file mode 100644
index cde33d6d7..000000000
--- a/app/workers/maintenance/destroy_media_worker.rb
+++ /dev/null
@@ -1,14 +0,0 @@
-# frozen_string_literal: true
-
-class Maintenance::DestroyMediaWorker
-  include Sidekiq::Worker
-
-  sidekiq_options queue: 'pull'
-
-  def perform(media_attachment_id)
-    media = media_attachment_id.is_a?(MediaAttachment) ? media_attachment_id : MediaAttachment.find(media_attachment_id)
-    media.destroy
-  rescue ActiveRecord::RecordNotFound
-    true
-  end
-end
diff --git a/app/workers/maintenance/redownload_account_media_worker.rb b/app/workers/maintenance/redownload_account_media_worker.rb
deleted file mode 100644
index 6afbe6e19..000000000
--- a/app/workers/maintenance/redownload_account_media_worker.rb
+++ /dev/null
@@ -1,16 +0,0 @@
-# frozen_string_literal: true
-
-class Maintenance::RedownloadAccountMediaWorker
-  include Sidekiq::Worker
-
-  sidekiq_options queue: 'pull', retry: false
-
-  def perform(account_id)
-    account = account_id.is_a?(Account) ? account_id : Account.find(account_id)
-    account.reset_avatar!
-    account.reset_header!
-    account.save
-  rescue ActiveRecord::RecordNotFound
-    true
-  end
-end
diff --git a/app/workers/maintenance/uncache_media_worker.rb b/app/workers/maintenance/uncache_media_worker.rb
deleted file mode 100644
index 4bc62ef75..000000000
--- a/app/workers/maintenance/uncache_media_worker.rb
+++ /dev/null
@@ -1,18 +0,0 @@
-# frozen_string_literal: true
-
-class Maintenance::UncacheMediaWorker
-  include Sidekiq::Worker
-
-  sidekiq_options queue: 'pull'
-
-  def perform(media_attachment_id)
-    media = media_attachment_id.is_a?(MediaAttachment) ? media_attachment_id : MediaAttachment.find(media_attachment_id)
-
-    return if media.file.blank?
-
-    media.file.destroy
-    media.save
-  rescue ActiveRecord::RecordNotFound
-    true
-  end
-end
diff --git a/app/workers/maintenance/uncache_preview_worker.rb b/app/workers/maintenance/uncache_preview_worker.rb
deleted file mode 100644
index 810ffd8cc..000000000
--- a/app/workers/maintenance/uncache_preview_worker.rb
+++ /dev/null
@@ -1,18 +0,0 @@
-# frozen_string_literal: true
-
-class Maintenance::UncachePreviewWorker
-  include Sidekiq::Worker
-
-  sidekiq_options queue: 'pull'
-
-  def perform(preview_card_id)
-    preview_card = PreviewCard.find(preview_card_id)
-
-    return if preview_card.image.blank?
-
-    preview_card.image.destroy
-    preview_card.save
-  rescue ActiveRecord::RecordNotFound
-    true
-  end
-end
diff --git a/lib/mastodon/accounts_cli.rb b/lib/mastodon/accounts_cli.rb
index d1854acc0..b16bf2e38 100644
--- a/lib/mastodon/accounts_cli.rb
+++ b/lib/mastodon/accounts_cli.rb
@@ -7,6 +7,8 @@ require_relative 'cli_helper'
 
 module Mastodon
   class AccountsCLI < Thor
+    include CLIHelper
+
     def self.exit_on_failure?
       true
     end
@@ -26,18 +28,20 @@ module Mastodon
       if options[:all]
         processed = 0
         delay     = 0
+        scope     = Account.local.without_suspended
+        progress  = create_progress_bar(scope.count)
 
-        Account.local.without_suspended.find_in_batches do |accounts|
+        scope.find_in_batches do |accounts|
           accounts.each do |account|
             rotate_keys_for_account(account, delay)
+            progress.increment
             processed += 1
-            say('.', :green, false)
           end
 
           delay += 5.minutes
         end
 
-        say
+        progress.finish
         say("OK, rotated keys for #{processed} accounts", :green)
       elsif username.present?
         rotate_keys_for_account(Account.find_local(username))
@@ -206,6 +210,8 @@ module Mastodon
       say('OK', :green)
     end
 
+    option :concurrency, type: :numeric, default: 5, aliases: [:c]
+    option :verbose, type: :boolean, aliases: [:v]
     option :dry_run, type: :boolean
     desc 'cull', 'Remove remote accounts that no longer exist'
     long_desc <<-LONG_DESC
@@ -215,63 +221,45 @@ module Mastodon
 
       Accounts that have had confirmed activity within the last week
       are excluded from the checks.
-
-      Domains that are unreachable are not checked.
-
-      With the --dry-run option, no deletes will actually be carried
-      out.
     LONG_DESC
     def cull
       skip_threshold = 7.days.ago
-      culled         = 0
-      dry_run_culled = []
-      skip_domains   = Set.new
       dry_run        = options[:dry_run] ? ' (DRY RUN)' : ''
+      skip_domains   = Concurrent::Set.new
 
-      Account.remote.where(protocol: :activitypub).partitioned.find_each do |account|
-        next if account.updated_at >= skip_threshold || (account.last_webfingered_at.present? && account.last_webfingered_at >= skip_threshold)
+      processed, culled = parallelize_with_progress(Account.remote.where(protocol: :activitypub).partitioned) do |account|
+        next if account.updated_at >= skip_threshold || (account.last_webfingered_at.present? && account.last_webfingered_at >= skip_threshold) || skip_domains.include?(account.domain)
 
         code = 0
-        unless skip_domains.include?(account.domain)
-          begin
-            code = Request.new(:head, account.uri).perform(&:code)
-          rescue HTTP::ConnectionError
-            skip_domains << account.domain
-          rescue StandardError
-            next
-          end
+
+        begin
+          code = Request.new(:head, account.uri).perform(&:code)
+        rescue HTTP::ConnectionError
+          skip_domains << account.domain
         end
 
         if [404, 410].include?(code)
-          if options[:dry_run]
-            dry_run_culled << account.acct
-          else
-            SuspendAccountService.new.call(account, destroy: true)
-          end
-          culled += 1
-          say('+', :green, false)
+          SuspendAccountService.new.call(account, destroy: true) unless options[:dry_run]
+          1
         else
-          account.touch # Touch account even during dry run to avoid getting the account into the window again
-          say('.', nil, false)
+          # Touch account even during dry run to avoid getting the account into the window again
+          account.touch
         end
       end
 
-      say
-      say("Removed #{culled} accounts. #{skip_domains.size} servers skipped#{dry_run}", skip_domains.empty? ? :green : :yellow)
+      say("Visited #{processed} accounts, removed #{culled}#{dry_run}", :green)
 
       unless skip_domains.empty?
-        say('The following servers were not available during the check:', :yellow)
+        say('The following domains were not available during the check:', :yellow)
         skip_domains.each { |domain| say('    ' + domain) }
       end
-
-      unless dry_run_culled.empty?
-        say('The following accounts would have been deleted:', :green)
-        dry_run_culled.each { |account| say('    ' + account) }
-      end
     end
 
     option :all, type: :boolean
     option :domain
+    option :concurrency, type: :numeric, default: 5, aliases: [:c]
+    option :verbose, type: :boolean, aliases: [:v]
+    option :dry_run, type: :boolean
     desc 'refresh [USERNAME]', 'Fetch remote user data and files'
     long_desc <<-LONG_DESC
       Fetch remote user data and files for one or multiple accounts.
@@ -280,21 +268,23 @@ module Mastodon
       Through the --domain option, this can be narrowed down to a
       specific domain only. Otherwise, a single remote account must
       be specified with USERNAME.
-
-      All processing is done in the background through Sidekiq.
     LONG_DESC
     def refresh(username = nil)
+      dry_run = options[:dry_run] ? ' (DRY RUN)' : ''
+
       if options[:domain] || options[:all]
-        queued = 0
         scope  = Account.remote
         scope  = scope.where(domain: options[:domain]) if options[:domain]
 
-        scope.select(:id).reorder(nil).find_in_batches do |accounts|
-          Maintenance::RedownloadAccountMediaWorker.push_bulk(accounts.map(&:id))
-          queued += accounts.size
+        processed, = parallelize_with_progress(scope) do |account|
+          next if options[:dry_run]
+
+          account.reset_avatar!
+          account.reset_header!
+          account.save
         end
 
-        say("Scheduled refreshment of #{queued} accounts", :green, true)
+        say("Refreshed #{processed} accounts#{dry_run}", :green, true)
       elsif username.present?
         username, domain = username.split('@')
         account = Account.find_remote(username, domain)
@@ -304,76 +294,53 @@ module Mastodon
           exit(1)
         end
 
-        Maintenance::RedownloadAccountMediaWorker.perform_async(account.id)
-        say('OK', :green)
+        unless options[:dry_run]
+          account.reset_avatar!
+          account.reset_header!
+          account.save
+        end
+
+        say("OK#{dry_run}", :green)
       else
         say('No account(s) given', :red)
         exit(1)
       end
     end
 
-    desc 'follow ACCT', 'Make all local accounts follow account specified by ACCT'
-    long_desc <<-LONG_DESC
-      Make all local accounts follow another local account specified by ACCT.
-      ACCT should be the username only.
-    LONG_DESC
-    def follow(acct)
-      if acct.include? '@'
-        say('Target account name should not contain a target instance, since it has to be a local account.', :red)
-        exit(1)
-      end
-
-      target_account = ResolveAccountService.new.call(acct)
-      processed      = 0
-      failed         = 0
+    option :concurrency, type: :numeric, default: 5, aliases: [:c]
+    option :verbose, type: :boolean, aliases: [:v]
+    desc 'follow USERNAME', 'Make all local accounts follow account specified by USERNAME'
+    def follow(username)
+      target_account = Account.find_local(username)
 
       if target_account.nil?
-        say("Target account (#{acct}) could not be resolved", :red)
+        say('No such account', :red)
         exit(1)
       end
 
-      Account.local.without_suspended.find_each do |account|
-        begin
-          FollowService.new.call(account, target_account)
-          processed += 1
-          say('.', :green, false)
-        rescue StandardError
-          failed += 1
-          say('.', :red, false)
-        end
+      processed, = parallelize_with_progress(Account.local.without_suspended) do |account|
+        FollowService.new.call(account, target_account)
       end
 
-      say("OK, followed target from #{processed} accounts, skipped #{failed}", :green)
+      say("OK, followed target from #{processed} accounts", :green)
     end
 
+    option :concurrency, type: :numeric, default: 5, aliases: [:c]
+    option :verbose, type: :boolean, aliases: [:v]
     desc 'unfollow ACCT', 'Make all local accounts unfollow account specified by ACCT'
-    long_desc <<-LONG_DESC
-      Make all local accounts unfollow an account specified by ACCT. ACCT can be
-      a simple username, in case of a local user. It can also be in the format
-      username@domain, in case of a remote user.
-    LONG_DESC
     def unfollow(acct)
       target_account = Account.find_remote(*acct.split('@'))
-      processed      = 0
-      failed         = 0
 
       if target_account.nil?
-        say("Target account (#{acct}) was not found", :red)
+        say('No such account', :red)
         exit(1)
       end
 
-      target_account.followers.local.find_each do |account|
-        begin
-          UnfollowService.new.call(account, target_account)
-          processed += 1
-          say('.', :green, false)
-        rescue StandardError
-          failed += 1
-          say('.', :red, false)
-        end
+      parallelize_with_progress(target_account.followers.local) do |account|
+        UnfollowService.new.call(account, target_account)
       end
 
-      say("OK, unfollowed target from #{processed} accounts, skipped #{failed}", :green)
+      say("OK, unfollowed target from #{processed} accounts", :green)
     end
 
     option :follows, type: :boolean, default: false
@@ -396,51 +363,50 @@ module Mastodon
       account = Account.find_local(username)
 
       if account.nil?
-        say('No user with such username', :red)
+        say('No such account', :red)
         exit(1)
       end
 
-      if options[:follows]
-        processed = 0
-        failed    = 0
+      total     = 0
+      total    += Account.where(id: ::Follow.where(account: account).select(:target_account_id)).count if options[:follows]
+      total    += Account.where(id: ::Follow.where(target_account: account).select(:account_id)).count if options[:followers]
+      progress  = create_progress_bar(total)
+      processed = 0
 
-        say("Unfollowing #{account.username}'s followees, this might take a while...")
+      if options[:follows]
+        scope = Account.where(id: ::Follow.where(account: account).select(:target_account_id))
 
-        Account.where(id: ::Follow.where(account: account).select(:target_account_id)).find_each do |target_account|
+        scope.find_each do |target_account|
           begin
             UnfollowService.new.call(account, target_account)
+          rescue => e
+            progress.log pastel.red("Error processing #{target_account.id}: #{e}")
+          ensure
+            progress.increment
             processed += 1
-            say('.', :green, false)
-          rescue StandardError
-            failed += 1
-            say('.', :red, false)
           end
         end
 
         BootstrapTimelineWorker.perform_async(account.id)
-
-        say("OK, unfollowed #{processed} followees, skipped #{failed}", :green)
       end
 
       if options[:followers]
-        processed = 0
-        failed    = 0
-
-        say("Removing #{account.username}'s followers, this might take a while...")
+        scope = Account.where(id: ::Follow.where(target_account: account).select(:account_id))
 
-        Account.where(id: ::Follow.where(target_account: account).select(:account_id)).find_each do |target_account|
+        scope.find_each do |target_account|
           begin
             UnfollowService.new.call(target_account, account)
+          rescue => e
+            progress.log pastel.red("Error processing #{target_account.id}: #{e}")
+          ensure
+            progress.increment
             processed += 1
-            say('.', :green, false)
-          rescue StandardError
-            failed += 1
-            say('.', :red, false)
           end
         end
-
-        say("OK, removed #{processed} followers, skipped #{failed}", :green)
       end
+
+      progress.finish
+      say("Processed #{processed} relationships", :green, true)
     end
 
     option :number, type: :numeric, aliases: [:n]
diff --git a/lib/mastodon/cache_cli.rb b/lib/mastodon/cache_cli.rb
index 5b0eea91b..803404c34 100644
--- a/lib/mastodon/cache_cli.rb
+++ b/lib/mastodon/cache_cli.rb
@@ -6,6 +6,8 @@ require_relative 'cli_helper'
 
 module Mastodon
   class CacheCLI < Thor
+    include CLIHelper
+
     def self.exit_on_failure?
       true
     end
@@ -16,6 +18,8 @@ module Mastodon
       say('OK', :green)
     end
 
+    option :concurrency, type: :numeric, default: 5, aliases: [:c]
+    option :verbose, type: :boolean, aliases: [:v]
     desc 'recount TYPE', 'Update hard-cached counters'
     long_desc <<~LONG_DESC
       Update hard-cached counters of TYPE by counting referenced
@@ -25,32 +29,24 @@ module Mastodon
       size of the database.
     LONG_DESC
     def recount(type)
-      processed = 0
-
       case type
       when 'accounts'
-        Account.local.includes(:account_stat).find_each do |account|
+        processed, = parallelize_with_progress(Account.local.includes(:account_stat)) do |account|
           account_stat                 = account.account_stat
           account_stat.following_count = account.active_relationships.count
           account_stat.followers_count = account.passive_relationships.count
           account_stat.statuses_count  = account.statuses.where.not(visibility: :direct).count
 
           account_stat.save if account_stat.changed?
-
-          processed += 1
-          say('.', :green, false)
         end
       when 'statuses'
-        Status.includes(:status_stat).find_each do |status|
+        processed, = parallelize_with_progress(Status.includes(:status_stat)) do |status|
           status_stat                  = status.status_stat
           status_stat.replies_count    = status.replies.where.not(visibility: :direct).count
           status_stat.reblogs_count    = status.reblogs.count
           status_stat.favourites_count = status.favourites.count
 
           status_stat.save if status_stat.changed?
-
-          processed += 1
-          say('.', :green, false)
         end
       else
         say("Unknown type: #{type}", :red)
diff --git a/lib/mastodon/cli_helper.rb b/lib/mastodon/cli_helper.rb
index 2f807d08c..da7348349 100644
--- a/lib/mastodon/cli_helper.rb
+++ b/lib/mastodon/cli_helper.rb
@@ -7,3 +7,52 @@ ActiveRecord::Base.logger    = dev_null
 ActiveJob::Base.logger       = dev_null
 HttpLog.configuration.logger = dev_null
 Paperclip.options[:log]      = false
+
+module Mastodon
+  module CLIHelper
+    def create_progress_bar(total = nil)
+      ProgressBar.create(total: total, format: '%c/%u |%b%i| %e')
+    end
+
+    def parallelize_with_progress(scope)
+      ActiveRecord::Base.configurations[Rails.env]['pool'] = options[:concurrency]
+
+      progress  = create_progress_bar(scope.count)
+      pool      = Concurrent::FixedThreadPool.new(options[:concurrency])
+      total     = Concurrent::AtomicFixnum.new(0)
+      aggregate = Concurrent::AtomicFixnum.new(0)
+
+      scope.reorder(nil).find_in_batches do |items|
+        futures = []
+
+        items.each do |item|
+          futures << Concurrent::Future.execute(executor: pool) do
+            ActiveRecord::Base.connection_pool.with_connection do
+              begin
+                progress.log("Processing #{item.id}") if options[:verbose]
+
+                result = yield(item)
+                aggregate.increment(result) if result.is_a?(Integer)
+              rescue => e
+                progress.log pastel.red("Error processing #{item.id}: #{e}")
+              ensure
+                progress.increment
+              end
+            end
+          end
+        end
+
+        total.increment(items.size)
+        futures.map(&:value)
+      end
+
+      progress.finish
+
+      [total.value, aggregate.value]
+    end
+
+    def pastel
+      @pastel ||= Pastel.new
+    end
+  end
+end
diff --git a/lib/mastodon/domains_cli.rb b/lib/mastodon/domains_cli.rb
index 17cafd1bc..c612c2d72 100644
--- a/lib/mastodon/domains_cli.rb
+++ b/lib/mastodon/domains_cli.rb
@@ -7,10 +7,14 @@ require_relative 'cli_helper'
 
 module Mastodon
   class DomainsCLI < Thor
+    include CLIHelper
+
     def self.exit_on_failure?
       true
     end
 
+    option :concurrency, type: :numeric, default: 5, aliases: [:c]
+    option :verbose, type: :boolean, aliases: [:v]
     option :dry_run, type: :boolean
     option :whitelist_mode, type: :boolean
     desc 'purge [DOMAIN]', 'Remove accounts from a DOMAIN without a trace'
@@ -24,7 +28,6 @@ module Mastodon
       are removed from the database.
     LONG_DESC
     def purge(domain = nil)
-      removed = 0
       dry_run = options[:dry_run] ? ' (DRY RUN)' : ''
 
       scope = begin
@@ -38,25 +41,22 @@ module Mastodon
         end
       end
 
-      scope.find_each do |account|
+      processed, = parallelize_with_progress(scope) do |account|
         SuspendAccountService.new.call(account, destroy: true) unless options[:dry_run]
-        removed += 1
-        say('.', :green, false)
       end
 
       DomainBlock.where(domain: domain).destroy_all unless options[:dry_run]
 
-      say
-      say("Removed #{removed} accounts#{dry_run}", :green)
+      say("Removed #{processed} accounts#{dry_run}", :green)
 
       custom_emojis = CustomEmoji.where(domain: domain)
       custom_emojis_count = custom_emojis.count
       custom_emojis.destroy_all unless options[:dry_run]
+
       say("Removed #{custom_emojis_count} custom emojis", :green)
     end
 
     option :concurrency, type: :numeric, default: 50, aliases: [:c]
-    option :silent, type: :boolean, default: false, aliases: [:s]
     option :format, type: :string, default: 'summary', aliases: [:f]
     option :exclude_suspended, type: :boolean, default: false, aliases: [:x]
     desc 'crawl [START]', 'Crawl all known peers, optionally beginning at START'
@@ -69,8 +69,6 @@ module Mastodon
       The --concurrency (-c) option controls the number of threads performing HTTP
       requests at the same time. More threads means the crawl may complete faster.
 
-      The --silent (-s) option controls progress output.
-
       The --format (-f) option controls how the data is displayed at the end. By
       default (`summary`), a summary of the statistics is returned. The other options
       are `domains`, which returns a newline-delimited list of all discovered peers,
@@ -87,6 +85,7 @@ module Mastodon
       start_at        = Time.now.to_f
       seed            = start ? [start] : Account.remote.domains
       blocked_domains = Regexp.new('\\.?' + DomainBlock.where(severity: 1).pluck(:domain).join('|') + '$')
+      progress        = create_progress_bar
 
       pool = Concurrent::ThreadPoolExecutor.new(min_threads: 0, max_threads: options[:concurrency], idletime: 10, auto_terminate: true, max_queue: 0)
 
@@ -95,7 +94,6 @@ module Mastodon
         next if options[:exclude_suspended] && domain.match(blocked_domains)
 
         stats[domain] = nil
-        processed.increment
 
         begin
           Request.new(:get, "https://#{domain}/api/v1/instance").perform do |res|
@@ -115,11 +113,11 @@ module Mastodon
             next unless res.code == 200
             stats[domain]['activity'] = Oj.load(res.to_s)
           end
-
-          say('.', :green, false) unless options[:silent]
         rescue StandardError
           failed.increment
-          say('.', :red, false) unless options[:silent]
+        ensure
+          processed.increment
+          progress.increment unless progress.finished?
         end
       end
 
@@ -133,10 +131,9 @@ module Mastodon
       pool.shutdown
       pool.wait_for_termination(20)
     ensure
+      progress.finish
       pool.shutdown
 
-      say unless options[:silent]
-
       case options[:format]
       when 'summary'
         stats_to_summary(stats, processed, failed, start_at)
diff --git a/lib/mastodon/feeds_cli.rb b/lib/mastodon/feeds_cli.rb
index fe11c3df4..ea7c90dff 100644
--- a/lib/mastodon/feeds_cli.rb
+++ b/lib/mastodon/feeds_cli.rb
@@ -6,55 +6,33 @@ require_relative 'cli_helper'
 
 module Mastodon
   class FeedsCLI < Thor
+    include CLIHelper
+
     def self.exit_on_failure?
       true
     end
 
     option :all, type: :boolean, default: false
-    option :background, type: :boolean, default: false
+    option :concurrency, type: :numeric, default: 5, aliases: [:c]
+    option :verbose, type: :boolean, aliases: [:v]
     option :dry_run, type: :boolean, default: false
-    option :verbose, type: :boolean, default: false
     desc 'build [USERNAME]', 'Build home and list feeds for one or all users'
     long_desc <<-LONG_DESC
       Build home and list feeds that are stored in Redis from the database.
 
       With the --all option, all active users will be processed.
       Otherwise, a single user specified by USERNAME.
-
-      With the --background option, regeneration will be queued into Sidekiq,
-      and the command will exit as soon as possible.
-
-      With the --dry-run option, no work will be done.
-
-      With the --verbose option, when accounts are processed sequentially in the
-      foreground, the IDs of the accounts will be printed.
     LONG_DESC
     def build(username = nil)
       dry_run = options[:dry_run] ? '(DRY RUN)' : ''
 
       if options[:all] || username.nil?
-        processed = 0
-        queued    = 0
 
-        User.active.select(:id, :account_id).reorder(nil).find_in_batches do |users|
-          if options[:background]
-            RegenerationWorker.push_bulk(users.map(&:account_id)) unless options[:dry_run]
-            queued += users.size
-          else
-            users.each do |user|
-              RegenerationWorker.new.perform(user.account_id) unless options[:dry_run]
-              options[:verbose] ? say(user.account_id) : say('.', :green, false)
-              processed += 1
-            end
-          end
+        processed, = parallelize_with_progress(Account.joins(:user).merge(User.active)) do |account|
+          PrecomputeFeedService.new.call(account) unless options[:dry_run]
         end
 
-        if options[:background]
-          say("Scheduled feed regeneration for #{queued} accounts #{dry_run}", :green, true)
-        else
-          say
-          say("Regenerated feeds for #{processed} accounts #{dry_run}", :green, true)
-        end
+        say("Regenerated feeds for #{processed} accounts #{dry_run}", :green, true)
       elsif username.present?
         account = Account.find_local(username)
 
@@ -63,11 +41,7 @@ module Mastodon
           exit(1)
         end
 
-        if options[:background]
-          RegenerationWorker.perform_async(account.id) unless options[:dry_run]
-        else
-          RegenerationWorker.new.perform(account.id) unless options[:dry_run]
-        end
+        PrecomputeFeedService.new.call(account) unless options[:dry_run]
 
         say("OK #{dry_run}", :green, true)
       else
diff --git a/lib/mastodon/media_cli.rb b/lib/mastodon/media_cli.rb
index 6152d5a09..0659b6b65 100644
--- a/lib/mastodon/media_cli.rb
+++ b/lib/mastodon/media_cli.rb
@@ -7,14 +7,15 @@ require_relative 'cli_helper'
 module Mastodon
   class MediaCLI < Thor
     include ActionView::Helpers::NumberHelper
+    include CLIHelper
 
     def self.exit_on_failure?
       true
     end
 
-    option :days, type: :numeric, default: 7
-    option :background, type: :boolean, default: false
-    option :verbose, type: :boolean, default: false
+    option :days, type: :numeric, default: 7, aliases: [:d]
+    option :concurrency, type: :numeric, default: 5, aliases: [:c]
+    option :verbose, type: :boolean, default: false, aliases: [:v]
     option :dry_run, type: :boolean, default: false
     desc 'remove', 'Remove remote media files'
     long_desc <<-DESC
@@ -22,49 +23,25 @@ module Mastodon
 
       The --days option specifies how old media attachments have to be before
       they are removed. It defaults to 7 days.
-
-      With the --background option, instead of deleting the files sequentially,
-      they will be queued into Sidekiq and the command will exit as soon as
-      possible. In Sidekiq they will be processed with higher concurrency, but
-      it may impact other operations of the Mastodon server, and it may overload
-      the underlying file storage.
-
-      With the --dry-run option, no work will be done.
-
-      With the --verbose option, when media attachments are processed sequentially in the
-      foreground, the IDs of the media attachments will be printed.
     DESC
     def remove
-      time_ago  = options[:days].days.ago
-      queued    = 0
-      processed = 0
-      size      = 0
-      dry_run   = options[:dry_run] ? '(DRY RUN)' : ''
+      time_ago = options[:days].days.ago
+      dry_run  = options[:dry_run] ? '(DRY RUN)' : ''
 
-      if options[:background]
-        MediaAttachment.where.not(remote_url: '').where.not(file_file_name: nil).where('created_at < ?', time_ago).select(:id, :file_file_size).reorder(nil).find_in_batches do |media_attachments|
-          queued += media_attachments.size
-          size   += media_attachments.reduce(0) { |sum, m| sum + (m.file_file_size || 0) }
-          Maintenance::UncacheMediaWorker.push_bulk(media_attachments.map(&:id)) unless options[:dry_run]
-        end
-      else
-        MediaAttachment.where.not(remote_url: '').where.not(file_file_name: nil).where('created_at < ?', time_ago).reorder(nil).find_in_batches do |media_attachments|
-          media_attachments.each do |m|
-            size += m.file_file_size || 0
-            Maintenance::UncacheMediaWorker.new.perform(m) unless options[:dry_run]
-            options[:verbose] ? say(m.id) : say('.', :green, false)
-            processed += 1
-          end
-        end
-      end
+      processed, aggregate = parallelize_with_progress(MediaAttachment.cached.where.not(remote_url: '').where('created_at < ?', time_ago)) do |media_attachment|
+        next if media_attachment.file.blank?
+
+        size = media_attachment.file_file_size
 
-      say
+        unless options[:dry_run]
+          media_attachment.file.destroy
+          media_attachment.save
+        end
 
-      if options[:background]
-        say("Scheduled the deletion of #{queued} media attachments (approx. #{number_to_human_size(size)}) #{dry_run}", :green, true)
-      else
-        say("Removed #{processed} media attachments (approx. #{number_to_human_size(size)}) #{dry_run}", :green, true)
+        size
       end
+
+      say("Removed #{processed} media attachments (approx. #{number_to_human_size(aggregate)}) #{dry_run}", :green, true)
     end
   end
 end
diff --git a/lib/mastodon/preview_cards_cli.rb b/lib/mastodon/preview_cards_cli.rb
index 465fe7d0b..cf4407250 100644
--- a/lib/mastodon/preview_cards_cli.rb
+++ b/lib/mastodon/preview_cards_cli.rb
@@ -8,87 +8,52 @@ require_relative 'cli_helper'
 module Mastodon
   class PreviewCardsCLI < Thor
     include ActionView::Helpers::NumberHelper
+    include CLIHelper
 
     def self.exit_on_failure?
       true
     end
 
     option :days, type: :numeric, default: 180
-    option :background, type: :boolean, default: false
-    option :verbose, type: :boolean, default: false
+    option :concurrency, type: :numeric, default: 5, aliases: [:c]
+    option :verbose, type: :boolean, aliases: [:v]
     option :dry_run, type: :boolean, default: false
     option :link, type: :boolean, default: false
     desc 'remove', 'Remove preview cards'
     long_desc <<-DESC
-      Removes locally thumbnails for previews.
+      Removes local thumbnails for preview cards.
 
       The --days option specifies how old preview cards have to be before
-      they are removed. It defaults to 180 days.
+      they are removed. It defaults to 180 days. Since preview cards will
+      not be re-fetched unless the link is re-posted after 2 weeks from
+      last time, it is not recommended to delete preview cards within the
+      last 14 days.
 
-      With the --background option, instead of deleting the files sequentially,
-      they will be queued into Sidekiq and the command will exit as soon as
-      possible. In Sidekiq they will be processed with higher concurrency, but
-      it may impact other operations of the Mastodon server, and it may overload
-      the underlying file storage.
-
-      With the --dry-run option, no work will be done.
-
-      With the --verbose option, when preview cards are processed sequentially in the
-      foreground, the IDs of the preview cards will be printed.
-
-      With the --link option, delete only link-type preview cards.
+      With the --link option, only link-type preview cards will be deleted,
+      leaving video and photo cards untouched.
     DESC
     def remove
-      prompt    = TTY::Prompt.new
-      time_ago  = options[:days].days.ago
-      queued    = 0
-      processed = 0
-      size      = 0
-      dry_run   = options[:dry_run] ? '(DRY RUN)' : ''
-      link      = options[:link] ? 'link-type ' : ''
-      scope     = PreviewCard.where.not(image_file_name: nil)
-      scope     = scope.where.not(image_file_name: '')
-      scope     = scope.where(type: :link) if options[:link]
-      scope     = scope.where('updated_at < ?', time_ago)
+      time_ago = options[:days].days.ago
+      dry_run  = options[:dry_run] ? ' (DRY RUN)' : ''
+      link     = options[:link] ? 'link-type ' : ''
+      scope    = PreviewCard.cached
+      scope    = scope.where(type: :link) if options[:link]
+      scope    = scope.where('updated_at < ?', time_ago)
 
-      if time_ago > 2.weeks.ago
-        prompt.say "\n"
-        prompt.say('The preview cards less than the past two weeks will not be re-acquired even when needed.')
-        prompt.say "\n"
+      processed, aggregate = parallelize_with_progress(scope) do |preview_card|
+        next if preview_card.image.blank?
 
-        unless prompt.yes?('Are you sure you want to delete the preview cards?', default: false)
-          prompt.say "\n"
-          prompt.warn 'Nothing execute. Bye!'
-          prompt.say "\n"
-          exit(1)
-        end
-      end
+        size = preview_card.image_file_size
 
-      if options[:background]
-        scope.select(:id, :image_file_size).reorder(nil).find_in_batches do |preview_cards|
-          queued += preview_cards.size
-          size   += preview_cards.reduce(0) { |sum, p| sum + (p.image_file_size || 0) }
-          Maintenance::UncachePreviewWorker.push_bulk(preview_cards.map(&:id)) unless options[:dry_run]
+        unless options[:dry_run]
+          preview_card.image.destroy
+          preview_card.save
         end
 
-      else
-        scope.select(:id, :image_file_size).reorder(nil).find_in_batches do |preview_cards|
-          preview_cards.each do |p|
-            size += p.image_file_size || 0
-            Maintenance::UncachePreviewWorker.new.perform(p.id) unless options[:dry_run]
-            options[:verbose] ? say(p.id) : say('.', :green, false)
-            processed += 1
-          end
-        end
+        size
       end
 
-      say
-
-      if options[:background]
-        say("Scheduled the deletion of #{queued} #{link}preview cards (approx. #{number_to_human_size(size)}) #{dry_run}", :green, true)
-      else
-        say("Removed #{processed} #{link}preview cards (approx. #{number_to_human_size(size)}) #{dry_run}", :green, true)
-      end
+      say("Removed #{processed} #{link}preview cards (approx. #{number_to_human_size(aggregate)})#{dry_run}", :green, true)
     end
   end
 end
-- 
cgit 


From 031ca25014e0ba88d3dcc3086947b41449a672e2 Mon Sep 17 00:00:00 2001
From: Eugen Rochko <eugen@zeonfederated.com>
Date: Tue, 10 Sep 2019 15:29:12 +0200
Subject: Add retry for failed media downloads and `tootctl media refresh`
 (#11775)

---
 app/lib/activitypub/activity/create.rb | 21 +++++++------
 app/models/concerns/remotable.rb       | 12 ++++----
 app/models/media_attachment.rb         |  2 +-
 app/workers/redownload_media_worker.rb | 19 ++++++++++++
 lib/mastodon/media_cli.rb              | 54 ++++++++++++++++++++++++++++++++++
 5 files changed, 92 insertions(+), 16 deletions(-)
 create mode 100644 app/workers/redownload_media_worker.rb

(limited to 'app/models')

diff --git a/app/lib/activitypub/activity/create.rb b/app/lib/activitypub/activity/create.rb
index 000b77df5..dea7fd43c 100644
--- a/app/lib/activitypub/activity/create.rb
+++ b/app/lib/activitypub/activity/create.rb
@@ -189,22 +189,25 @@ class ActivityPub::Activity::Create < ActivityPub::Activity
     media_attachments = []
 
     as_array(@object['attachment']).each do |attachment|
-      next if attachment['url'].blank?
+      next if attachment['url'].blank? || media_attachments.size >= 4
 
-      href             = Addressable::URI.parse(attachment['url']).normalize.to_s
-      media_attachment = MediaAttachment.create(account: @account, remote_url: href, description: attachment['name'].presence, focus: attachment['focalPoint'], blurhash: supported_blurhash?(attachment['blurhash']) ? attachment['blurhash'] : nil)
-      media_attachments << media_attachment
+      begin
+        href             = Addressable::URI.parse(attachment['url']).normalize.to_s
+        media_attachment = MediaAttachment.create(account: @account, remote_url: href, description: attachment['name'].presence, focus: attachment['focalPoint'], blurhash: supported_blurhash?(attachment['blurhash']) ? attachment['blurhash'] : nil)
+        media_attachments << media_attachment
 
-      next if unsupported_media_type?(attachment['mediaType']) || skip_download?
+        next if unsupported_media_type?(attachment['mediaType']) || skip_download?
 
-      media_attachment.file_remote_url = href
-      media_attachment.save
+        media_attachment.file_remote_url = href
+        media_attachment.save
+      rescue Mastodon::UnexpectedResponseError, HTTP::TimeoutError, HTTP::ConnectionError, OpenSSL::SSL::SSLError
+        RedownloadMediaWorker.perform_in(rand(30..600).seconds, media_attachment.id)
+      end
     end
 
     media_attachments
   rescue Addressable::URI::InvalidURIError => e
-    Rails.logger.debug e
-
+    Rails.logger.debug "Invalid URL in attachment: #{e}"
     media_attachments
   end
 
diff --git a/app/models/concerns/remotable.rb b/app/models/concerns/remotable.rb
index 9372a963b..082302619 100644
--- a/app/models/concerns/remotable.rb
+++ b/app/models/concerns/remotable.rb
@@ -4,7 +4,7 @@ module Remotable
   extend ActiveSupport::Concern
 
   class_methods do
-    def remotable_attachment(attachment_name, limit)
+    def remotable_attachment(attachment_name, limit, suppress_errors: true)
       attribute_name  = "#{attachment_name}_remote_url".to_sym
       method_name     = "#{attribute_name}=".to_sym
       alt_method_name = "reset_#{attachment_name}!".to_sym
@@ -22,7 +22,7 @@ module Remotable
 
         begin
           Request.new(:get, url).perform do |response|
-            next if response.code != 200
+            raise Mastodon::UnexpectedResponseError, response unless (200...300).cover?(response.code)
 
             content_type = parse_content_type(response.headers.get('content-type').last)
             extname      = detect_extname_from_content_type(content_type)
@@ -41,11 +41,11 @@ module Remotable
 
             self[attribute_name] = url if has_attribute?(attribute_name)
           end
-        rescue HTTP::TimeoutError, HTTP::ConnectionError, OpenSSL::SSL::SSLError, Paperclip::Errors::NotIdentifiedByImageMagickError, Addressable::URI::InvalidURIError, Mastodon::HostValidationError, Mastodon::LengthValidationError => e
+        rescue Mastodon::UnexpectedResponseError, HTTP::TimeoutError, HTTP::ConnectionError, OpenSSL::SSL::SSLError => e
+          Rails.logger.debug "Error fetching remote #{attachment_name}: #{e}"
+          raise e unless suppress_errors
+        rescue Paperclip::Errors::NotIdentifiedByImageMagickError, Addressable::URI::InvalidURIError, Mastodon::HostValidationError, Mastodon::LengthValidationError, Paperclip::Error, Mastodon::DimensionsValidationError => e
           Rails.logger.debug "Error fetching remote #{attachment_name}: #{e}"
-          nil
-        rescue Paperclip::Error, Mastodon::DimensionsValidationError => e
-          Rails.logger.debug "Error processing remote #{attachment_name}: #{e}"
           nil
         end
       end
diff --git a/app/models/media_attachment.rb b/app/models/media_attachment.rb
index b58025015..a2b73f150 100644
--- a/app/models/media_attachment.rb
+++ b/app/models/media_attachment.rb
@@ -118,7 +118,7 @@ class MediaAttachment < ApplicationRecord
   validates_attachment_content_type :file, content_type: IMAGE_MIME_TYPES + VIDEO_MIME_TYPES + AUDIO_MIME_TYPES
   validates_attachment_size :file, less_than: IMAGE_LIMIT, unless: :larger_media_format?
   validates_attachment_size :file, less_than: VIDEO_LIMIT, if: :larger_media_format?
-  remotable_attachment :file, VIDEO_LIMIT
+  remotable_attachment :file, VIDEO_LIMIT, suppress_errors: false
 
   include Attachmentable
 
diff --git a/app/workers/redownload_media_worker.rb b/app/workers/redownload_media_worker.rb
new file mode 100644
index 000000000..98e995918
--- /dev/null
+++ b/app/workers/redownload_media_worker.rb
@@ -0,0 +1,19 @@
+# frozen_string_literal: true
+
+class RedownloadMediaWorker
+  include Sidekiq::Worker
+  include ExponentialBackoff
+
+  sidekiq_options queue: 'pull', retry: 3
+
+  def perform(id)
+    media_attachment = MediaAttachment.find(id)
+
+    return if media_attachment.remote_url.blank?
+
+    media_attachment.reset_file!
+    media_attachment.save
+  rescue ActiveRecord::RecordNotFound
+    true
+  end
+end
diff --git a/lib/mastodon/media_cli.rb b/lib/mastodon/media_cli.rb
index 0659b6b65..ec2f36c30 100644
--- a/lib/mastodon/media_cli.rb
+++ b/lib/mastodon/media_cli.rb
@@ -43,5 +43,59 @@ module Mastodon
 
       say("Removed #{processed} media attachments (approx. #{number_to_human_size(aggregate)}) #{dry_run}", :green, true)
     end
+
+    option :account, type: :string
+    option :domain, type: :string
+    option :status, type: :numeric
+    option :concurrency, type: :numeric, default: 5, aliases: [:c]
+    option :verbose, type: :boolean, default: false, aliases: [:v]
+    option :dry_run, type: :boolean, default: false
+    desc 'refresh', 'Fetch remote media files'
+    long_desc <<-DESC
+      Re-downloads media attachments from other servers. You must specify the
+      source of media attachments with one of the following options:
+
+      Use the --status option to download attachments from a specific status,
+      using the status local numeric ID.
+
+      Use the --account option to download attachments from a specific account,
+      using username@domain handle of the account.
+
+      Use the --domain option to download attachments from a specific domain.
+    DESC
+    def refresh
+      dry_run = options[:dry_run] ? ' (DRY RUN)' : ''
+
+      if options[:status]
+        scope = MediaAttachment.where(status_id: options[:status])
+      elsif options[:account]
+        username, domain = username.split('@')
+        account = Account.find_remote(username, domain)
+
+        if account.nil?
+          say('No such account', :red)
+          exit(1)
+        end
+
+        scope = MediaAttachment.where(account_id: account.id)
+      elsif options[:domain]
+        scope = MediaAttachment.joins(:account).merge(Account.by_domain_and_subdomains(options[:domain]))
+      else
+        exit(1)
+      end
+
+      processed, aggregate = parallelize_with_progress(scope) do |media_attachment|
+        next if media_attachment.remote_url.blank?
+
+        unless options[:dry_run]
+          media_attachment.reset_file!
+          media_attachment.save
+        end
+
+        media_attachment.file_file_size
+      end
+
+      say("Downloaded #{processed} media attachments (approx. #{number_to_human_size(aggregate)})#{dry_run}", :green, true)
+    end
   end
 end
-- 
cgit 


From 4fe127664b0ae22a528b4a4467ab2de92e3da3ef Mon Sep 17 00:00:00 2001
From: Tao Bror Bojlén <brortao@users.noreply.github.com>
Date: Wed, 11 Sep 2019 07:44:58 +0100
Subject: add admin setting for default search engine indexing (fix #11750)
 (#11804)

---
 app/lib/settings/scoped_settings.rb             | 1 +
 app/models/form/admin_settings.rb               | 2 ++
 app/views/admin/settings/edit.html.haml         | 3 +++
 config/locales/en.yml                           | 3 +++
 spec/controllers/application_controller_spec.rb | 1 +
 5 files changed, 10 insertions(+)

(limited to 'app/models')

diff --git a/app/lib/settings/scoped_settings.rb b/app/lib/settings/scoped_settings.rb
index 3653ab114..9ca39510a 100644
--- a/app/lib/settings/scoped_settings.rb
+++ b/app/lib/settings/scoped_settings.rb
@@ -4,6 +4,7 @@ module Settings
   class ScopedSettings
     DEFAULTING_TO_UNSCOPED = %w(
       theme
+      noindex
     ).freeze
 
     def initialize(object)
diff --git a/app/models/form/admin_settings.rb b/app/models/form/admin_settings.rb
index 6bc3ca9f5..24196e182 100644
--- a/app/models/form/admin_settings.rb
+++ b/app/models/form/admin_settings.rb
@@ -32,6 +32,7 @@ class Form::AdminSettings
     trends
     show_domain_blocks
     show_domain_blocks_rationale
+    noindex
   ).freeze
 
   BOOLEAN_KEYS = %i(
@@ -45,6 +46,7 @@ class Form::AdminSettings
     profile_directory
     spam_check_enabled
     trends
+    noindex
   ).freeze
 
   UPLOAD_KEYS = %i(
diff --git a/app/views/admin/settings/edit.html.haml b/app/views/admin/settings/edit.html.haml
index 28880c087..752386b3c 100644
--- a/app/views/admin/settings/edit.html.haml
+++ b/app/views/admin/settings/edit.html.haml
@@ -71,6 +71,9 @@
     .fields-group
       = f.input :trends, as: :boolean, wrapper: :with_label, label: t('admin.settings.trends.title'), hint: t('admin.settings.trends.desc_html')
 
+    .fields-group
+      = f.input :noindex, as: :boolean, wrapper: :with_label, label: t('admin.settings.default_noindex.title'), hint: t('admin.settings.default_noindex.desc_html')
+
   .fields-group
     = f.input :spam_check_enabled, as: :boolean, wrapper: :with_label, label: t('admin.settings.spam_check_enabled.title'), hint: t('admin.settings.spam_check_enabled.desc_html')
 
diff --git a/config/locales/en.yml b/config/locales/en.yml
index 52cb4a269..0a5ca31c1 100644
--- a/config/locales/en.yml
+++ b/config/locales/en.yml
@@ -427,6 +427,9 @@ en:
       custom_css:
         desc_html: Modify the look with CSS loaded on every page
         title: Custom CSS
+      default_noindex:
+        desc_html: Affects all users who have not changed this setting themselves
+        title: Opt users out of search engine indexing by default
       domain_blocks:
         all: To everyone
         disabled: To no one
diff --git a/spec/controllers/application_controller_spec.rb b/spec/controllers/application_controller_spec.rb
index 1811500df..da4a794cd 100644
--- a/spec/controllers/application_controller_spec.rb
+++ b/spec/controllers/application_controller_spec.rb
@@ -110,6 +110,7 @@ describe ApplicationController, type: :controller do
       sign_in current_user
 
       allow(Setting).to receive(:[]).with('theme').and_return 'contrast'
+      allow(Setting).to receive(:[]).with('noindex').and_return false
 
       expect(controller.view_context.current_theme).to eq 'contrast'
     end
-- 
cgit 


From c5d37f18cb3f4d6212fb8f3e1c4e1e027f677ec5 Mon Sep 17 00:00:00 2001
From: Eugen Rochko <eugen@zeonfederated.com>
Date: Wed, 11 Sep 2019 16:32:44 +0200
Subject: Change deletes to preserve soft-deleted statuses in unresolved
 reports (#11805)

Change all account actions except "none" to resolve all unresolved reports

Refactor `SuspendAccountService` to be more readable
---
 app/controllers/admin/accounts_controller.rb       |  2 +-
 app/controllers/admin/report_notes_controller.rb   |  9 ++--
 .../api/v1/admin/accounts_controller.rb            |  2 +-
 app/lib/activitypub/activity/delete.rb             |  3 +-
 app/models/account.rb                              |  1 +
 app/models/admin/account_action.rb                 | 24 +++++++--
 app/models/form/account_batch.rb                   |  2 +-
 app/models/form/status_batch.rb                    |  2 +-
 app/models/report.rb                               |  1 +
 app/models/status.rb                               |  4 ++
 app/models/user.rb                                 |  4 ++
 app/services/block_domain_service.rb               |  2 +-
 app/services/remove_status_service.rb              |  7 +--
 app/services/suspend_account_service.rb            | 62 ++++++++++++++++------
 app/services/unallow_domain_service.rb             |  2 +-
 app/workers/admin/suspension_worker.rb             |  2 +-
 lib/mastodon/accounts_cli.rb                       |  4 +-
 lib/mastodon/domains_cli.rb                        |  2 +-
 .../admin/reported_statuses_controller_spec.rb     |  2 +-
 spec/controllers/admin/statuses_controller_spec.rb |  2 +-
 spec/models/form/status_batch_spec.rb              |  4 +-
 21 files changed, 98 insertions(+), 45 deletions(-)

(limited to 'app/models')

diff --git a/app/controllers/admin/accounts_controller.rb b/app/controllers/admin/accounts_controller.rb
index 2fa1dfe5f..68b6352f8 100644
--- a/app/controllers/admin/accounts_controller.rb
+++ b/app/controllers/admin/accounts_controller.rb
@@ -41,7 +41,7 @@ module Admin
 
     def reject
       authorize @account.user, :reject?
-      SuspendAccountService.new.call(@account, including_user: true, destroy: true, skip_distribution: true)
+      SuspendAccountService.new.call(@account, reserve_email: false, reserve_username: false)
       redirect_to admin_pending_accounts_path
     end
 
diff --git a/app/controllers/admin/report_notes_controller.rb b/app/controllers/admin/report_notes_controller.rb
index bcb3f2026..b816c5b5d 100644
--- a/app/controllers/admin/report_notes_controller.rb
+++ b/app/controllers/admin/report_notes_controller.rb
@@ -5,10 +5,10 @@ module Admin
     before_action :set_report_note, only: [:destroy]
 
     def create
-      authorize ReportNote, :create?
+      authorize :report_note, :create?
 
       @report_note = current_account.report_notes.new(resource_params)
-      @report = @report_note.report
+      @report      = @report_note.report
 
       if @report_note.save
         if params[:create_and_resolve]
@@ -26,9 +26,8 @@ module Admin
 
         redirect_to admin_report_path(@report), notice: I18n.t('admin.report_notes.created_msg')
       else
-        @report_notes = @report.notes.latest
-        @report_history = @report.history
-        @form = Form::StatusBatch.new
+        @report_notes = (@report.notes.latest + @report.history + @report.target_account.targeted_account_warnings.latest.custom).sort_by(&:created_at)
+        @form         = Form::StatusBatch.new
 
         render template: 'admin/reports/show'
       end
diff --git a/app/controllers/api/v1/admin/accounts_controller.rb b/app/controllers/api/v1/admin/accounts_controller.rb
index c306180ca..c35ea5ab2 100644
--- a/app/controllers/api/v1/admin/accounts_controller.rb
+++ b/app/controllers/api/v1/admin/accounts_controller.rb
@@ -58,7 +58,7 @@ class Api::V1::Admin::AccountsController < Api::BaseController
 
   def reject
     authorize @account.user, :reject?
-    SuspendAccountService.new.call(@account, including_user: true, destroy: true, skip_distribution: true)
+    SuspendAccountService.new.call(@account, reserve_email: false, reserve_username: false)
     render json: @account, serializer: REST::Admin::AccountSerializer
   end
 
diff --git a/app/lib/activitypub/activity/delete.rb b/app/lib/activitypub/activity/delete.rb
index 345060462..dc9ff580c 100644
--- a/app/lib/activitypub/activity/delete.rb
+++ b/app/lib/activitypub/activity/delete.rb
@@ -13,8 +13,7 @@ class ActivityPub::Activity::Delete < ActivityPub::Activity
 
   def delete_person
     lock_or_return("delete_in_progress:#{@account.id}") do
-      SuspendAccountService.new.call(@account)
-      @account.destroy!
+      SuspendAccountService.new.call(@account, reserve_username: false)
     end
   end
 
diff --git a/app/models/account.rb b/app/models/account.rb
index 8c9388b95..55fe53fae 100644
--- a/app/models/account.rb
+++ b/app/models/account.rb
@@ -115,6 +115,7 @@ class Account < ApplicationRecord
            :approved?,
            :pending?,
            :disabled?,
+           :unconfirmed_or_pending?,
            :role,
            :admin?,
            :moderator?,
diff --git a/app/models/admin/account_action.rb b/app/models/admin/account_action.rb
index c7da8b52c..b30a82369 100644
--- a/app/models/admin/account_action.rb
+++ b/app/models/admin/account_action.rb
@@ -83,19 +83,23 @@ class Admin::AccountAction
 
     # A log entry is only interesting if the warning contains
     # custom text from someone. Otherwise it's just noise.
+
     log_action(:create, warning) if warning.text.present?
   end
 
   def process_reports!
-    return if report_id.blank?
+    # If we're doing "mark as resolved" on a single report,
+    # then we want to keep other reports open in case they
+    # contain new actionable information.
+    #
+    # Otherwise, we will mark all unresolved reports about
+    # the account as resolved.
 
-    authorize(report, :update?)
+    reports.each { |report| authorize(report, :update?) }
 
-    if type == 'none'
+    reports.each do |report|
       log_action(:resolve, report)
       report.resolve!(current_account)
-    else
-      Report.where(target_account: target_account).unresolved.update_all(action_taken: true, action_taken_by_account_id: current_account.id)
     end
   end
 
@@ -141,6 +145,16 @@ class Admin::AccountAction
     @report.status_ids if @report && include_statuses
   end
 
+  def reports
+    @reports ||= begin
+      if type == 'none' && with_report?
+        [report]
+      else
+        Report.where(target_account: target_account).unresolved
+      end
+    end
+  end
+
   def warning_preset
     @warning_preset ||= AccountWarningPreset.find(warning_preset_id) if warning_preset_id.present?
   end
diff --git a/app/models/form/account_batch.rb b/app/models/form/account_batch.rb
index f1b7a4566..0b285fde9 100644
--- a/app/models/form/account_batch.rb
+++ b/app/models/form/account_batch.rb
@@ -69,6 +69,6 @@ class Form::AccountBatch
     records = accounts.includes(:user)
 
     records.each { |account| authorize(account.user, :reject?) }
-           .each { |account| SuspendAccountService.new.call(account, including_user: true, destroy: true, skip_distribution: true) }
+           .each { |account| SuspendAccountService.new.call(account, reserve_email: false, reserve_username: false) }
   end
 end
diff --git a/app/models/form/status_batch.rb b/app/models/form/status_batch.rb
index e09cc2594..c4943a7ea 100644
--- a/app/models/form/status_batch.rb
+++ b/app/models/form/status_batch.rb
@@ -35,7 +35,7 @@ class Form::StatusBatch
   def delete_statuses
     Status.where(id: status_ids).reorder(nil).find_each do |status|
       status.discard
-      RemovalWorker.perform_async(status.id, redraft: false)
+      RemovalWorker.perform_async(status.id, immediate: true)
       Tombstone.find_or_create_by(uri: status.uri, account: status.account, by_moderator: true)
       log_action :destroy, status
     end
diff --git a/app/models/report.rb b/app/models/report.rb
index 1e707ff1c..fb2e040ee 100644
--- a/app/models/report.rb
+++ b/app/models/report.rb
@@ -59,6 +59,7 @@ class Report < ApplicationRecord
   end
 
   def resolve!(acting_account)
+    RemovalWorker.push_bulk(Status.with_discarded.discarded.where(id: status_ids).pluck(:id)) { |status_id| [status_id, { immediate: true }] }
     update!(action_taken: true, action_taken_by_account_id: acting_account.id)
   end
 
diff --git a/app/models/status.rb b/app/models/status.rb
index 9cfaddcec..471bb03b4 100644
--- a/app/models/status.rb
+++ b/app/models/status.rb
@@ -214,6 +214,10 @@ class Status < ApplicationRecord
     !sensitive? && with_media?
   end
 
+  def reported?
+    @reported ||= Report.where(target_account: account).unresolved.where('? = ANY(status_ids)', id).exists?
+  end
+
   def emojis
     return @emojis if defined?(@emojis)
 
diff --git a/app/models/user.rb b/app/models/user.rb
index 95f1d8fc5..78b82a68f 100644
--- a/app/models/user.rb
+++ b/app/models/user.rb
@@ -171,6 +171,10 @@ class User < ApplicationRecord
     confirmed? && approved? && !disabled? && !account.suspended?
   end
 
+  def unconfirmed_or_pending?
+    !(confirmed? && approved?)
+  end
+
   def inactive_message
     !approved? ? :pending : super
   end
diff --git a/app/services/block_domain_service.rb b/app/services/block_domain_service.rb
index 0ec6be503..ae461abf2 100644
--- a/app/services/block_domain_service.rb
+++ b/app/services/block_domain_service.rb
@@ -53,7 +53,7 @@ class BlockDomainService < BaseService
 
   def suspend_accounts!
     blocked_domain_accounts.without_suspended.reorder(nil).find_each do |account|
-      SuspendAccountService.new.call(account, suspended_at: @domain_block.created_at)
+      SuspendAccountService.new.call(account, reserve_username: true, suspended_at: @domain_block.created_at)
     end
   end
 
diff --git a/app/services/remove_status_service.rb b/app/services/remove_status_service.rb
index 685c1d4bf..f9352ed3d 100644
--- a/app/services/remove_status_service.rb
+++ b/app/services/remove_status_service.rb
@@ -8,7 +8,8 @@ class RemoveStatusService < BaseService
   # @param   [Status] status
   # @param   [Hash] options
   # @option  [Boolean] :redraft
-  # @options [Boolean] :original_removed
+  # @option  [Boolean] :immediate
+  # @option [Boolean] :original_removed
   def call(status, **options)
     @payload  = Oj.dump(event: :delete, payload: status.id.to_s)
     @status   = status
@@ -31,7 +32,7 @@ class RemoveStatusService < BaseService
         remove_from_spam_check
         remove_media
 
-        @status.destroy!
+        @status.destroy! if @options[:immediate] || !@status.reported?
       else
         raise Mastodon::RaceConditionError
       end
@@ -150,7 +151,7 @@ class RemoveStatusService < BaseService
   end
 
   def remove_media
-    return if @options[:redraft]
+    return if @options[:redraft] || (!@options[:immediate] && @status.reported?)
 
     @status.media_attachments.destroy_all
   end
diff --git a/app/services/suspend_account_service.rb b/app/services/suspend_account_service.rb
index 85da7e921..ecc893931 100644
--- a/app/services/suspend_account_service.rb
+++ b/app/services/suspend_account_service.rb
@@ -15,7 +15,6 @@ class SuspendAccountService < BaseService
     favourites
     follow_requests
     list_accounts
-    media_attachments
     mute_relationships
     muted_by_relationships
     notifications
@@ -32,14 +31,26 @@ class SuspendAccountService < BaseService
     targeted_reports
   ).freeze
 
-  # Suspend an account and remove as much of its data as possible
+  # Suspend or remove an account and remove as much of its data
+  # as possible. If it's a local account and it has not been confirmed
+  # or never been approved, then side effects are skipped and both
+  # the user and account records are removed fully. Otherwise,
+  # it is controlled by options.
   # @param [Account]
   # @param [Hash] options
-  # @option [Boolean] :including_user Remove the user record as well
-  # @option [Boolean] :destroy Remove the account record instead of suspending
+  # @option [Boolean] :reserve_email Keep user record. Only applicable for local accounts
+  # @option [Boolean] :reserve_username Keep account record
+  # @option [Boolean] :skip_side_effects Side effects are ActivityPub and streaming API payloads
+  # @option [Time]    :suspended_at Only applicable when :reserve_username is true
   def call(account, **options)
     @account = account
-    @options = options
+    @options = { reserve_username: true, reserve_email: true }.merge(options)
+
+    if @account.local? && @account.user_unconfirmed_or_pending?
+      @options[:reserve_email]     = false
+      @options[:reserve_username]  = false
+      @options[:skip_side_effects] = true
+    end
 
     reject_follows!
     purge_user!
@@ -60,27 +71,39 @@ class SuspendAccountService < BaseService
   def purge_user!
     return if !@account.local? || @account.user.nil?
 
-    if @options[:including_user]
-      @options[:destroy] = true if !@account.user_confirmed? || @account.user_pending?
-      @account.user.destroy
-    else
+    if @options[:reserve_email]
       @account.user.disable!
       @account.user.invites.where(uses: 0).destroy_all
+    else
+      @account.user.destroy
     end
   end
 
   def purge_content!
-    distribute_delete_actor! if @account.local? && !@options[:skip_distribution]
+    distribute_delete_actor! if @account.local? && !@options[:skip_side_effects]
 
     @account.statuses.reorder(nil).find_in_batches do |statuses|
-      BatchedRemoveStatusService.new.call(statuses, skip_side_effects: @options[:destroy])
+      statuses.reject! { |status| reported_status_ids.include?(status.id) } if @options[:reserve_username]
+      BatchedRemoveStatusService.new.call(statuses, skip_side_effects: @options[:skip_side_effects])
+    end
+
+    @account.media_attachments.reorder(nil).find_each do |media_attachment|
+      next if @options[:reserve_username] && reported_status_ids.include?(media_attachment.status_id)
+
+      media_attachment.destroy
+    end
+
+    @account.polls.reorder(nil).find_each do |poll|
+      next if @options[:reserve_username] && reported_status_ids.include?(poll.status_id)
+
+      poll.destroy
     end
 
     associations_for_destruction.each do |association_name|
       destroy_all(@account.public_send(association_name))
     end
 
-    @account.destroy if @options[:destroy]
+    @account.destroy unless @options[:reserve_username]
   end
 
   def purge_profile!
@@ -88,11 +111,13 @@ class SuspendAccountService < BaseService
     # there is no point wasting time updating
     # its values first
 
-    return if @options[:destroy]
+    return unless @options[:reserve_username]
 
     @account.silenced_at      = nil
     @account.suspended_at     = @options[:suspended_at] || Time.now.utc
     @account.locked           = false
+    @account.memorial         = false
+    @account.discoverable     = false
     @account.display_name     = ''
     @account.note             = ''
     @account.fields           = []
@@ -100,6 +125,7 @@ class SuspendAccountService < BaseService
     @account.followers_count  = 0
     @account.following_count  = 0
     @account.moved_to_account = nil
+    @account.trust_level      = :untrusted
     @account.avatar.destroy
     @account.header.destroy
     @account.save!
@@ -135,11 +161,15 @@ class SuspendAccountService < BaseService
     Account.inboxes - delivery_inboxes
   end
 
+  def reported_status_ids
+    @reported_status_ids ||= Report.where(target_account: @account).unresolved.pluck(:status_ids).flatten.uniq
+  end
+
   def associations_for_destruction
-    if @options[:destroy]
-      ASSOCIATIONS_ON_SUSPEND + ASSOCIATIONS_ON_DESTROY
-    else
+    if @options[:reserve_username]
       ASSOCIATIONS_ON_SUSPEND
+    else
+      ASSOCIATIONS_ON_SUSPEND + ASSOCIATIONS_ON_DESTROY
     end
   end
 end
diff --git a/app/services/unallow_domain_service.rb b/app/services/unallow_domain_service.rb
index d4387c1a1..bd1ad328d 100644
--- a/app/services/unallow_domain_service.rb
+++ b/app/services/unallow_domain_service.rb
@@ -3,7 +3,7 @@
 class UnallowDomainService < BaseService
   def call(domain_allow)
     Account.where(domain: domain_allow.domain).find_each do |account|
-      SuspendAccountService.new.call(account, destroy: true)
+      SuspendAccountService.new.call(account, reserve_username: false)
     end
 
     domain_allow.destroy
diff --git a/app/workers/admin/suspension_worker.rb b/app/workers/admin/suspension_worker.rb
index ae8b24d8c..83c815efd 100644
--- a/app/workers/admin/suspension_worker.rb
+++ b/app/workers/admin/suspension_worker.rb
@@ -6,6 +6,6 @@ class Admin::SuspensionWorker
   sidekiq_options queue: 'pull'
 
   def perform(account_id, remove_user = false)
-    SuspendAccountService.new.call(Account.find(account_id), including_user: remove_user)
+    SuspendAccountService.new.call(Account.find(account_id), reserve_username: true, reserve_email: !remove_user)
   end
 end
diff --git a/lib/mastodon/accounts_cli.rb b/lib/mastodon/accounts_cli.rb
index b16bf2e38..a09a6ab04 100644
--- a/lib/mastodon/accounts_cli.rb
+++ b/lib/mastodon/accounts_cli.rb
@@ -185,7 +185,7 @@ module Mastodon
       end
 
       say("Deleting user with #{account.statuses_count} statuses, this might take a while...")
-      SuspendAccountService.new.call(account, including_user: true)
+      SuspendAccountService.new.call(account, reserve_email: false)
       say('OK', :green)
     end
 
@@ -239,7 +239,7 @@ module Mastodon
         end
 
         if [404, 410].include?(code)
-          SuspendAccountService.new.call(account, destroy: true) unless options[:dry_run]
+          SuspendAccountService.new.call(account, reserve_username: false) unless options[:dry_run]
           1
         else
           # Touch account even during dry run to avoid getting the account into the window again
diff --git a/lib/mastodon/domains_cli.rb b/lib/mastodon/domains_cli.rb
index c612c2d72..8e52de1c3 100644
--- a/lib/mastodon/domains_cli.rb
+++ b/lib/mastodon/domains_cli.rb
@@ -42,7 +42,7 @@ module Mastodon
       end
 
       processed, = parallelize_with_progress(scope) do |account|
-        SuspendAccountService.new.call(account, destroy: true) unless options[:dry_run]
+        SuspendAccountService.new.call(account, reserve_username: false, skip_side_effects: true) unless options[:dry_run]
       end
 
       DomainBlock.where(domain: domain).destroy_all unless options[:dry_run]
diff --git a/spec/controllers/admin/reported_statuses_controller_spec.rb b/spec/controllers/admin/reported_statuses_controller_spec.rb
index bd146b795..2a1598123 100644
--- a/spec/controllers/admin/reported_statuses_controller_spec.rb
+++ b/spec/controllers/admin/reported_statuses_controller_spec.rb
@@ -47,7 +47,7 @@ describe Admin::ReportedStatusesController do
       it 'removes a status' do
         allow(RemovalWorker).to receive(:perform_async)
         subject.call
-        expect(RemovalWorker).to have_received(:perform_async).with(status_ids.first, redraft: false)
+        expect(RemovalWorker).to have_received(:perform_async).with(status_ids.first, immediate: true)
       end
     end
 
diff --git a/spec/controllers/admin/statuses_controller_spec.rb b/spec/controllers/admin/statuses_controller_spec.rb
index 6b06343ef..d9690d83f 100644
--- a/spec/controllers/admin/statuses_controller_spec.rb
+++ b/spec/controllers/admin/statuses_controller_spec.rb
@@ -65,7 +65,7 @@ describe Admin::StatusesController do
       it 'removes a status' do
         allow(RemovalWorker).to receive(:perform_async)
         subject.call
-        expect(RemovalWorker).to have_received(:perform_async).with(status_ids.first, redraft: false)
+        expect(RemovalWorker).to have_received(:perform_async).with(status_ids.first, immediate: true)
       end
     end
 
diff --git a/spec/models/form/status_batch_spec.rb b/spec/models/form/status_batch_spec.rb
index f9c58c90f..68d84a737 100644
--- a/spec/models/form/status_batch_spec.rb
+++ b/spec/models/form/status_batch_spec.rb
@@ -41,12 +41,12 @@ describe Form::StatusBatch do
 
     it 'call RemovalWorker' do
       form.save
-      expect(RemovalWorker).to have_received(:perform_async).with(status.id, redraft: false)
+      expect(RemovalWorker).to have_received(:perform_async).with(status.id, immediate: true)
     end
 
     it 'do not call RemovalWorker' do
       form.save
-      expect(RemovalWorker).not_to have_received(:perform_async).with(another_status.id, redraft: false)
+      expect(RemovalWorker).not_to have_received(:perform_async).with(another_status.id, immediate: true)
     end
   end
 end
-- 
cgit 


From b6381bdc7dcfe5713b99e2aae0491115350c7724 Mon Sep 17 00:00:00 2001
From: Eugen Rochko <eugen@zeonfederated.com>
Date: Fri, 13 Sep 2019 16:00:34 +0200
Subject: Change max length of media descriptions from 420 to 1500 characters
 (#11819)

Fix #11658
---
 app/javascript/mastodon/features/ui/components/focal_point_modal.js | 2 +-
 app/models/media_attachment.rb                                      | 2 +-
 2 files changed, 2 insertions(+), 2 deletions(-)

(limited to 'app/models')

diff --git a/app/javascript/mastodon/features/ui/components/focal_point_modal.js b/app/javascript/mastodon/features/ui/components/focal_point_modal.js
index 735e445e8..d13138a76 100644
--- a/app/javascript/mastodon/features/ui/components/focal_point_modal.js
+++ b/app/javascript/mastodon/features/ui/components/focal_point_modal.js
@@ -223,7 +223,7 @@ class FocalPointModal extends ImmutablePureComponent {
 
             <div className='setting-text__toolbar'>
               <button disabled={detecting || media.get('type') !== 'image'} className='link-button' onClick={this.handleTextDetection}><FormattedMessage id='upload_modal.detect_text' defaultMessage='Detect text from picture' /></button>
-              <CharacterCounter max={420} text={detecting ? '' : description} />
+              <CharacterCounter max={1500} text={detecting ? '' : description} />
             </div>
 
             <Button disabled={!dirty || detecting || length(description) > 420} text={intl.formatMessage(messages.apply)} onClick={this.handleSubmit} />
diff --git a/app/models/media_attachment.rb b/app/models/media_attachment.rb
index a2b73f150..c4932f2ef 100644
--- a/app/models/media_attachment.rb
+++ b/app/models/media_attachment.rb
@@ -123,7 +123,7 @@ class MediaAttachment < ApplicationRecord
   include Attachmentable
 
   validates :account, presence: true
-  validates :description, length: { maximum: 420 }, if: :local?
+  validates :description, length: { maximum: 1_500 }, if: :local?
 
   scope :attached,   -> { where.not(status_id: nil).or(where.not(scheduled_status_id: nil)) }
   scope :unattached, -> { where(status_id: nil, scheduled_status_id: nil) }
-- 
cgit 


From 59da5ccb8e9e06e0c91ebd548e899786c632f1de Mon Sep 17 00:00:00 2001
From: Eugen Rochko <eugen@zeonfederated.com>
Date: Fri, 13 Sep 2019 16:01:09 +0200
Subject: Change unlisted custom emoji to not appear in autosuggestions
 (#11818)

Fix #11669
---
 app/controllers/api/v1/custom_emojis_controller.rb | 2 +-
 app/models/custom_emoji.rb                         | 5 +++--
 2 files changed, 4 insertions(+), 3 deletions(-)

(limited to 'app/models')

diff --git a/app/controllers/api/v1/custom_emojis_controller.rb b/app/controllers/api/v1/custom_emojis_controller.rb
index 252f667dd..4e6d5d7c6 100644
--- a/app/controllers/api/v1/custom_emojis_controller.rb
+++ b/app/controllers/api/v1/custom_emojis_controller.rb
@@ -7,6 +7,6 @@ class Api::V1::CustomEmojisController < Api::BaseController
 
   def index
     expires_in 3.minutes, public: true
-    render_with_cache(each_serializer: REST::CustomEmojiSerializer) { CustomEmoji.local.where(disabled: false).includes(:category) }
+    render_with_cache(each_serializer: REST::CustomEmojiSerializer) { CustomEmoji.listed.includes(:category) }
   end
 end
diff --git a/app/models/custom_emoji.rb b/app/models/custom_emoji.rb
index 0a4201a14..edb1bec75 100644
--- a/app/models/custom_emoji.rb
+++ b/app/models/custom_emoji.rb
@@ -40,10 +40,11 @@ class CustomEmoji < ApplicationRecord
   validates_attachment :image, content_type: { content_type: IMAGE_MIME_TYPES }, presence: true, size: { less_than: LIMIT }
   validates :shortcode, uniqueness: { scope: :domain }, format: { with: /\A#{SHORTCODE_RE_FRAGMENT}\z/ }, length: { minimum: 2 }
 
-  scope :local,      -> { where(domain: nil) }
-  scope :remote,     -> { where.not(domain: nil) }
+  scope :local, -> { where(domain: nil) }
+  scope :remote, -> { where.not(domain: nil) }
   scope :alphabetic, -> { order(domain: :asc, shortcode: :asc) }
   scope :by_domain_and_subdomains, ->(domain) { where(domain: domain).or(where(arel_table[:domain].matches('%.' + domain))) }
+  scope :listed, -> { local.where(disabled: false).where(visible_in_picker: true) }
 
   remotable_attachment :image, LIMIT
 
-- 
cgit 


From 0762258aec68f1d422a0ecdc29d637c5807f943a Mon Sep 17 00:00:00 2001
From: Eugen Rochko <eugen@zeonfederated.com>
Date: Fri, 13 Sep 2019 16:01:26 +0200
Subject: Fix hashtags being split by ZWNJ character (#11821)

Fix #11761
---
 app/models/tag.rb       | 5 +++--
 spec/models/tag_spec.rb | 4 ++++
 2 files changed, 7 insertions(+), 2 deletions(-)

(limited to 'app/models')

diff --git a/app/models/tag.rb b/app/models/tag.rb
index 135e0a030..a6aed0d68 100644
--- a/app/models/tag.rb
+++ b/app/models/tag.rb
@@ -25,8 +25,9 @@ class Tag < ApplicationRecord
   has_many :featured_tags, dependent: :destroy, inverse_of: :tag
   has_one :account_tag_stat, dependent: :destroy
 
-  HASHTAG_NAME_RE = '([[:word:]_][[:word:]_·]*[[:alpha:]_·][[:word:]_·]*[[:word:]_])|([[:word:]_]*[[:alpha:]][[:word:]_]*)'
-  HASHTAG_RE = /(?:^|[^\/\)\w])#(#{HASHTAG_NAME_RE})/i
+  HASHTAG_SEPARATORS = "_\u00B7\u200c"
+  HASHTAG_NAME_RE    = "([[:word:]_][[:word:]#{HASHTAG_SEPARATORS}]*[[:alpha:]#{HASHTAG_SEPARATORS}][[:word:]#{HASHTAG_SEPARATORS}]*[[:word:]_])|([[:word:]_]*[[:alpha:]][[:word:]_]*)"
+  HASHTAG_RE         = /(?:^|[^\/\)\w])#(#{HASHTAG_NAME_RE})/i
 
   validates :name, presence: true, format: { with: /\A(#{HASHTAG_NAME_RE})\z/i }
   validate :validate_name_change, if: -> { !new_record? && name_changed? }
diff --git a/spec/models/tag_spec.rb b/spec/models/tag_spec.rb
index 2bb30fb57..df876593c 100644
--- a/spec/models/tag_spec.rb
+++ b/spec/models/tag_spec.rb
@@ -62,6 +62,10 @@ RSpec.describe Tag, type: :model do
       expect(subject.match('hello #one·two·three').to_s).to eq ' #one·two·three'
     end
 
+    it 'matches ZWNJ' do
+      expect(subject.match('just add #نرم‌افزار and').to_s).to eq ' #نرم‌افزار'
+    end
+
     it 'does not match middle dots at the start' do
       expect(subject.match('hello #·one·two·three')).to be_nil
     end
-- 
cgit 


From c707ef49d9b13932f4d98c127ec3148a5cdc3479 Mon Sep 17 00:00:00 2001
From: Eugen Rochko <eugen@zeonfederated.com>
Date: Sun, 15 Sep 2019 21:08:39 +0200
Subject: Fix 2FA challenge and password challenge for non-database users
 (#11831)

* Fix 2FA challenge not appearing for non-database users

Fix #11685

* Fix account deletion not working when using external login

Fix #11691
---
 app/controllers/auth/sessions_controller.rb       | 61 ++++++++++-------------
 app/controllers/settings/deletes_controller.rb    | 25 +++++++---
 app/models/form/delete_confirmation.rb            |  2 +-
 app/views/settings/deletes/show.html.haml         |  5 +-
 config/initializers/devise.rb                     |  7 ++-
 config/locales/en.yml                             |  3 +-
 spec/controllers/auth/sessions_controller_spec.rb | 24 +++------
 7 files changed, 66 insertions(+), 61 deletions(-)

(limited to 'app/models')

diff --git a/app/controllers/auth/sessions_controller.rb b/app/controllers/auth/sessions_controller.rb
index 7e6dbf19e..3e93b2e68 100644
--- a/app/controllers/auth/sessions_controller.rb
+++ b/app/controllers/auth/sessions_controller.rb
@@ -8,8 +8,6 @@ class Auth::SessionsController < Devise::SessionsController
   skip_before_action :require_no_authentication, only: [:create]
   skip_before_action :require_functional!
 
-  prepend_before_action :authenticate_with_two_factor, if: :two_factor_enabled?, only: [:create]
-
   before_action :set_instance_presenter, only: [:new]
   before_action :set_body_classes
 
@@ -22,9 +20,22 @@ class Auth::SessionsController < Devise::SessionsController
   end
 
   def create
-    super do |resource|
-      remember_me(resource)
-      flash.delete(:notice)
+    self.resource = begin
+      if user_params[:email].blank? && session[:otp_user_id].present?
+        User.find(session[:otp_user_id])
+      else
+        warden.authenticate!(auth_options)
+      end
+    end
+
+    if resource.otp_required_for_login?
+      if user_params[:otp_attempt].present? && session[:otp_user_id].present?
+        authenticate_with_two_factor_via_otp(resource)
+      else
+        prompt_for_two_factor(resource)
+      end
+    else
+      authenticate_and_respond(resource)
     end
   end
 
@@ -37,18 +48,6 @@ class Auth::SessionsController < Devise::SessionsController
 
   protected
 
-  def find_user
-    if session[:otp_user_id]
-      User.find(session[:otp_user_id])
-    elsif user_params[:email]
-      if use_seamless_external_login? && Devise.check_at_sign && user_params[:email].index('@').nil?
-        User.joins(:account).find_by(accounts: { username: user_params[:email] })
-      else
-        User.find_for_authentication(email: user_params[:email])
-      end
-    end
-  end
-
   def user_params
     params.require(:user).permit(:email, :password, :otp_attempt)
   end
@@ -71,32 +70,17 @@ class Auth::SessionsController < Devise::SessionsController
     super
   end
 
-  def two_factor_enabled?
-    find_user.try(:otp_required_for_login?)
-  end
-
   def valid_otp_attempt?(user)
     user.validate_and_consume_otp!(user_params[:otp_attempt]) ||
       user.invalidate_otp_backup_code!(user_params[:otp_attempt])
-  rescue OpenSSL::Cipher::CipherError => _error
+  rescue OpenSSL::Cipher::CipherError
     false
   end
 
-  def authenticate_with_two_factor
-    user = self.resource = find_user
-
-    if user_params[:otp_attempt].present? && session[:otp_user_id]
-      authenticate_with_two_factor_via_otp(user)
-    elsif user&.valid_password?(user_params[:password])
-      prompt_for_two_factor(user)
-    end
-  end
-
   def authenticate_with_two_factor_via_otp(user)
     if valid_otp_attempt?(user)
       session.delete(:otp_user_id)
-      remember_me(user)
-      sign_in(user)
+      authenticate_and_respond(user)
     else
       flash.now[:alert] = I18n.t('users.invalid_otp_token')
       prompt_for_two_factor(user)
@@ -108,6 +92,13 @@ class Auth::SessionsController < Devise::SessionsController
     render :two_factor
   end
 
+  def authenticate_and_respond(user)
+    sign_in(user)
+    remember_me(user)
+
+    respond_with user, location: after_sign_in_path_for(user)
+  end
+
   private
 
   def set_instance_presenter
@@ -120,9 +111,11 @@ class Auth::SessionsController < Devise::SessionsController
 
   def home_paths(resource)
     paths = [about_path]
+
     if single_user_mode? && resource.is_a?(User)
       paths << short_account_path(username: resource.account)
     end
+
     paths
   end
 
diff --git a/app/controllers/settings/deletes_controller.rb b/app/controllers/settings/deletes_controller.rb
index 97fe4d328..15a59c999 100644
--- a/app/controllers/settings/deletes_controller.rb
+++ b/app/controllers/settings/deletes_controller.rb
@@ -14,12 +14,11 @@ class Settings::DeletesController < Settings::BaseController
   end
 
   def destroy
-    if current_user.valid_password?(delete_params[:password])
-      Admin::SuspensionWorker.perform_async(current_user.account_id, true)
-      sign_out
+    if challenge_passed?
+      destroy_account!
       redirect_to new_user_session_path, notice: I18n.t('deletes.success_msg')
     else
-      redirect_to settings_delete_path, alert: I18n.t('deletes.bad_password_msg')
+      redirect_to settings_delete_path, alert: I18n.t('deletes.challenge_not_passed')
     end
   end
 
@@ -29,11 +28,25 @@ class Settings::DeletesController < Settings::BaseController
     redirect_to root_path unless Setting.open_deletion
   end
 
-  def delete_params
-    params.require(:form_delete_confirmation).permit(:password)
+  def resource_params
+    params.require(:form_delete_confirmation).permit(:password, :username)
   end
 
   def require_not_suspended!
     forbidden if current_account.suspended?
   end
+
+  def challenge_passed?
+    if current_user.encrypted_password.blank?
+      current_account.username == resource_params[:username]
+    else
+      current_user.valid_password?(resource_params[:password])
+    end
+  end
+
+  def destroy_account!
+    current_account.suspend!
+    Admin::SuspensionWorker.perform_async(current_user.account_id, true)
+    sign_out
+  end
 end
diff --git a/app/models/form/delete_confirmation.rb b/app/models/form/delete_confirmation.rb
index 0884a09b8..99d04b331 100644
--- a/app/models/form/delete_confirmation.rb
+++ b/app/models/form/delete_confirmation.rb
@@ -3,5 +3,5 @@
 class Form::DeleteConfirmation
   include ActiveModel::Model
 
-  attr_accessor :password
+  attr_accessor :password, :username
 end
diff --git a/app/views/settings/deletes/show.html.haml b/app/views/settings/deletes/show.html.haml
index 6e2ff31c5..08792e0af 100644
--- a/app/views/settings/deletes/show.html.haml
+++ b/app/views/settings/deletes/show.html.haml
@@ -20,7 +20,10 @@
 
   %hr.spacer/
 
-  = f.input :password, wrapper: :with_block_label, input_html: { :autocomplete => 'off' }, hint: t('deletes.confirm_password')
+  - if current_user.encrypted_password.present?
+    = f.input :password, wrapper: :with_block_label, input_html: { :autocomplete => 'off' }, hint: t('deletes.confirm_password')
+  - else
+    = f.input :username, wrapper: :with_block_label, input_html: { :autocomplete => 'off' }, hint: t('deletes.confirm_username')
 
   .actions
     = f.button :button, t('deletes.proceed'), type: :submit, class: 'negative'
diff --git a/config/initializers/devise.rb b/config/initializers/devise.rb
index cd9bacf68..311583820 100644
--- a/config/initializers/devise.rb
+++ b/config/initializers/devise.rb
@@ -71,10 +71,13 @@ end
 
 Devise.setup do |config|
   config.warden do |manager|
+    manager.default_strategies(scope: :user).unshift :database_authenticatable
     manager.default_strategies(scope: :user).unshift :ldap_authenticatable if Devise.ldap_authentication
     manager.default_strategies(scope: :user).unshift :pam_authenticatable  if Devise.pam_authentication
-    manager.default_strategies(scope: :user).unshift :two_factor_authenticatable
-    manager.default_strategies(scope: :user).unshift :two_factor_backupable
+
+    # We handle 2FA in our own sessions controller so this gets in the way
+    manager.default_strategies(scope: :user).delete :two_factor_backupable
+    manager.default_strategies(scope: :user).delete :two_factor_authenticatable
   end
 
   # The secret key used by Devise. Devise uses this key to generate
diff --git a/config/locales/en.yml b/config/locales/en.yml
index 0a5ca31c1..8c9fe89f8 100644
--- a/config/locales/en.yml
+++ b/config/locales/en.yml
@@ -632,8 +632,9 @@ en:
       x_months: "%{count}mo"
       x_seconds: "%{count}s"
   deletes:
-    bad_password_msg: The password you entered was incorrect
+    challenge_not_passed: The information you entered was not correct
     confirm_password: Enter your current password to verify your identity
+    confirm_username: Enter your username to confirm the procedure
     proceed: Delete account
     success_msg: Your account was successfully deleted
     warning:
diff --git a/spec/controllers/auth/sessions_controller_spec.rb b/spec/controllers/auth/sessions_controller_spec.rb
index 87ef4f2bb..7ed5edde0 100644
--- a/spec/controllers/auth/sessions_controller_spec.rb
+++ b/spec/controllers/auth/sessions_controller_spec.rb
@@ -5,11 +5,11 @@ require 'rails_helper'
 RSpec.describe Auth::SessionsController, type: :controller do
   render_views
 
-  describe 'GET #new' do
-    before do
-      request.env['devise.mapping'] = Devise.mappings[:user]
-    end
+  before do
+    request.env['devise.mapping'] = Devise.mappings[:user]
+  end
 
+  describe 'GET #new' do
     it 'returns http success' do
       get :new
       expect(response).to have_http_status(200)
@@ -19,10 +19,6 @@ RSpec.describe Auth::SessionsController, type: :controller do
   describe 'DELETE #destroy' do
     let(:user) { Fabricate(:user) }
 
-    before do
-      request.env['devise.mapping'] = Devise.mappings[:user]
-    end
-
     context 'with a regular user' do
       it 'redirects to home after sign out' do
         sign_in(user, scope: :user)
@@ -51,10 +47,6 @@ RSpec.describe Auth::SessionsController, type: :controller do
   end
 
   describe 'POST #create' do
-    before do
-      request.env['devise.mapping'] = Devise.mappings[:user]
-    end
-
     context 'using PAM authentication', if: ENV['PAM_ENABLED'] == 'true' do
       context 'using a valid password' do
         before do
@@ -191,11 +183,11 @@ RSpec.describe Auth::SessionsController, type: :controller do
     end
 
     context 'using two-factor authentication' do
-      let(:user) do
-        Fabricate(:user, email: 'x@y.com', password: 'abcdefgh',
-                         otp_required_for_login: true, otp_secret: User.generate_otp_secret(32))
+      let!(:user) do
+        Fabricate(:user, email: 'x@y.com', password: 'abcdefgh', otp_required_for_login: true, otp_secret: User.generate_otp_secret(32))
       end
-      let(:recovery_codes) do
+
+      let!(:recovery_codes) do
         codes = user.generate_otp_backup_codes!
         user.save
         return codes
-- 
cgit 


From ef0d22f232723be035e95bde13310d02bf1c127b Mon Sep 17 00:00:00 2001
From: mayaeh <mayaeh@marimo-net.org>
Date: Mon, 16 Sep 2019 21:27:29 +0900
Subject: Add search and sort functions to hashtag admin UI (#11829)

* Add search and sort functions to hashtag admin UI

* Move scope processing from tags_controller to tag_filter

* Fix based on method naming conventions

* Fixed not to get 500 errors for invalid requests
---
 app/controllers/admin/tags_controller.rb | 15 +++--------
 app/helpers/admin/filter_helper.rb       |  2 +-
 app/models/tag.rb                        |  1 +
 app/models/tag_filter.rb                 | 44 ++++++++++++++++++++++++++++++++
 app/views/admin/tags/index.html.haml     | 32 ++++++++++++++++++-----
 config/locales/en.yml                    |  4 +++
 config/locales/simple_form.en.yml        |  2 ++
 7 files changed, 81 insertions(+), 19 deletions(-)
 create mode 100644 app/models/tag_filter.rb

(limited to 'app/models')

diff --git a/app/controllers/admin/tags_controller.rb b/app/controllers/admin/tags_controller.rb
index 376ebe44d..65341bbfb 100644
--- a/app/controllers/admin/tags_controller.rb
+++ b/app/controllers/admin/tags_controller.rb
@@ -2,7 +2,6 @@
 
 module Admin
   class TagsController < BaseController
-    before_action :set_tags, only: :index
     before_action :set_tag, except: [:index, :batch, :approve_all, :reject_all]
     before_action :set_usage_by_domain, except: [:index, :batch, :approve_all, :reject_all]
     before_action :set_counters, except: [:index, :batch, :approve_all, :reject_all]
@@ -10,6 +9,7 @@ module Admin
     def index
       authorize :tag, :index?
 
+      @tags = filtered_tags.page(params[:page])
       @form = Form::TagBatch.new
     end
 
@@ -48,10 +48,6 @@ module Admin
 
     private
 
-    def set_tags
-      @tags = filtered_tags.page(params[:page])
-    end
-
     def set_tag
       @tag = Tag.find(params[:id])
     end
@@ -73,16 +69,11 @@ module Admin
     end
 
     def filtered_tags
-      scope = Tag
-      scope = scope.discoverable if filter_params[:context] == 'directory'
-      scope = scope.unreviewed if filter_params[:review] == 'unreviewed'
-      scope = scope.reviewed.order(reviewed_at: :desc) if filter_params[:review] == 'reviewed'
-      scope = scope.pending_review.order(requested_review_at: :desc) if filter_params[:review] == 'pending_review'
-      scope.order(max_score: :desc)
+      TagFilter.new(filter_params).results
     end
 
     def filter_params
-      params.slice(:context, :review, :page).permit(:context, :review, :page)
+      params.slice(:directory, :reviewed, :unreviewed, :pending_review, :page, :popular, :active, :name).permit(:directory, :reviewed, :unreviewed, :pending_review, :page, :popular, :active, :name)
     end
 
     def tag_params
diff --git a/app/helpers/admin/filter_helper.rb b/app/helpers/admin/filter_helper.rb
index 506429e10..8af1683e7 100644
--- a/app/helpers/admin/filter_helper.rb
+++ b/app/helpers/admin/filter_helper.rb
@@ -5,7 +5,7 @@ module Admin::FilterHelper
   REPORT_FILTERS       = %i(resolved account_id target_account_id).freeze
   INVITE_FILTER        = %i(available expired).freeze
   CUSTOM_EMOJI_FILTERS = %i(local remote by_domain shortcode).freeze
-  TAGS_FILTERS         = %i(context review).freeze
+  TAGS_FILTERS         = %i(directory reviewed unreviewed pending_review popular active name).freeze
   INSTANCES_FILTERS    = %i(limited by_domain).freeze
   FOLLOWERS_FILTERS    = %i(relationship status by_domain activity order).freeze
 
diff --git a/app/models/tag.rb b/app/models/tag.rb
index a6aed0d68..4e77c404d 100644
--- a/app/models/tag.rb
+++ b/app/models/tag.rb
@@ -39,6 +39,7 @@ class Tag < ApplicationRecord
   scope :listable, -> { where(listable: [true, nil]) }
   scope :discoverable, -> { listable.joins(:account_tag_stat).where(AccountTagStat.arel_table[:accounts_count].gt(0)).order(Arel.sql('account_tag_stats.accounts_count desc')) }
   scope :most_used, ->(account) { joins(:statuses).where(statuses: { account: account }).group(:id).order(Arel.sql('count(*) desc')) }
+  scope :matches_name, ->(value) { where(arel_table[:name].matches("#{value}%")) }
 
   delegate :accounts_count,
            :accounts_count=,
diff --git a/app/models/tag_filter.rb b/app/models/tag_filter.rb
new file mode 100644
index 000000000..8921e186b
--- /dev/null
+++ b/app/models/tag_filter.rb
@@ -0,0 +1,44 @@
+# frozen_string_literal: true
+
+class TagFilter
+  attr_reader :params
+
+  def initialize(params)
+    @params = params
+  end
+
+  def results
+    scope = Tag.unscoped
+
+    params.each do |key, value|
+      next if key.to_s == 'page'
+
+      scope.merge!(scope_for(key, value.to_s.strip)) if value.present?
+    end
+
+    scope.order(id: :desc)
+  end
+
+  private
+
+  def scope_for(key, value)
+    case key.to_s
+    when 'directory'
+      Tag.discoverable
+    when 'reviewed'
+      Tag.reviewed.order(reviewed_at: :desc)
+    when 'unreviewed'
+      Tag.unreviewed
+    when 'pending_review'
+      Tag.pending_review.order(requested_review_at: :desc)
+    when 'popular'
+      Tag.order('max_score DESC NULLS LAST')
+    when 'active'
+      Tag.order('last_status_at DESC NULLS LAST')
+    when 'name'
+      Tag.matches_name(value)
+    else
+      raise "Unknown filter: #{key}"
+    end
+  end
+end
diff --git a/app/views/admin/tags/index.html.haml b/app/views/admin/tags/index.html.haml
index 324d13d3e..cea1b71b5 100644
--- a/app/views/admin/tags/index.html.haml
+++ b/app/views/admin/tags/index.html.haml
@@ -8,16 +8,36 @@
   .filter-subset
     %strong= t('admin.tags.context')
     %ul
-      %li= filter_link_to t('generic.all'), context: nil
-      %li= filter_link_to t('admin.tags.directory'), context: 'directory'
+      %li= filter_link_to t('generic.all'), directory: nil
+      %li= filter_link_to t('admin.tags.directory'), directory: '1'
 
   .filter-subset
     %strong= t('admin.tags.review')
     %ul
-      %li= filter_link_to t('generic.all'), review: nil
-      %li= filter_link_to t('admin.tags.unreviewed'), review: 'unreviewed'
-      %li= filter_link_to t('admin.tags.reviewed'), review: 'reviewed'
-      %li= filter_link_to safe_join([t('admin.accounts.moderation.pending'), "(#{Tag.pending_review.count})"], ' '), review: 'pending_review'
+      %li= filter_link_to t('generic.all'), reviewed: nil, unreviewed: nil, pending_review: nil
+      %li= filter_link_to t('admin.tags.unreviewed'), unreviewed: '1', reviewed: nil, pending_review: nil
+      %li= filter_link_to t('admin.tags.reviewed'), reviewed: '1', unreviewed: nil, pending_review: nil
+      %li= filter_link_to safe_join([t('admin.accounts.moderation.pending'), "(#{Tag.pending_review.count})"], ' '), pending_review: '1', reviewed: nil, unreviewed: nil
+
+  .filter-subset
+    %strong= t('generic.order_by')
+    %ul
+      %li= filter_link_to t('admin.tags.most_recent'), popular: nil, active: nil
+      %li= filter_link_to t('admin.tags.most_popular'), popular: '1', active: nil
+      %li= filter_link_to t('admin.tags.last_active'), active: '1', popular: nil
+
+= form_tag admin_tags_url, method: 'GET', class: 'simple_form' do
+  .fields-group
+    - Admin::FilterHelper::TAGS_FILTERS.each do |key|
+      = hidden_field_tag key, params[key] if params[key].present?
+
+    - %i(name).each do |key|
+      .input.string.optional
+        = text_field_tag key, params[key], class: 'string optional', placeholder: I18n.t("admin.tags.#{key}")
+
+    .actions
+      %button= t('admin.accounts.search')
+      = link_to t('admin.accounts.reset'), admin_tags_path, class: 'button negative'
 
 %hr.spacer/
 
diff --git a/config/locales/en.yml b/config/locales/en.yml
index 8c9fe89f8..f05fdd48b 100644
--- a/config/locales/en.yml
+++ b/config/locales/en.yml
@@ -521,6 +521,10 @@ en:
       context: Context
       directory: In directory
       in_directory: "%{count} in directory"
+      last_active: Last active
+      most_popular: Most popular
+      most_recent: Most recent
+      name: Hashtag
       review: Review status
       reviewed: Reviewed
       title: Hashtags
diff --git a/config/locales/simple_form.en.yml b/config/locales/simple_form.en.yml
index 2e5982de9..c542377a9 100644
--- a/config/locales/simple_form.en.yml
+++ b/config/locales/simple_form.en.yml
@@ -131,6 +131,8 @@ en:
         must_be_follower: Block notifications from non-followers
         must_be_following: Block notifications from people you don't follow
         must_be_following_dm: Block direct messages from people you don't follow
+      invite:
+        comment: Comment
       invite_request:
         text: Why do you want to join?
       notification_emails:
-- 
cgit 


From 3d14b8f28f96eb4da352f615ee5b9c06ecdcc269 Mon Sep 17 00:00:00 2001
From: Eugen Rochko <eugen@zeonfederated.com>
Date: Mon, 16 Sep 2019 20:42:29 +0200
Subject: Fix SSO login not using existing account when e-mail is verified
 (#11862)

Fix #11472
---
 app/models/concerns/omniauthable.rb | 30 +++++++++++++++++-------------
 1 file changed, 17 insertions(+), 13 deletions(-)

(limited to 'app/models')

diff --git a/app/models/concerns/omniauthable.rb b/app/models/concerns/omniauthable.rb
index b9c124841..960784222 100644
--- a/app/models/concerns/omniauthable.rb
+++ b/app/models/concerns/omniauthable.rb
@@ -4,7 +4,7 @@ module Omniauthable
   extend ActiveSupport::Concern
 
   TEMP_EMAIL_PREFIX = 'change@me'
-  TEMP_EMAIL_REGEX = /\Achange@me/
+  TEMP_EMAIL_REGEX  = /\A#{TEMP_EMAIL_PREFIX}/.freeze
 
   included do
     devise :omniauthable
@@ -28,8 +28,8 @@ module Omniauthable
       # to prevent the identity being locked with accidentally created accounts.
       # Note that this may leave zombie accounts (with no associated identity) which
       # can be cleaned up at a later date.
-      user = signed_in_resource || identity.user
-      user = create_for_oauth(auth) if user.nil?
+      user   = signed_in_resource || identity.user
+      user ||= create_for_oauth(auth)
 
       if identity.user.nil?
         identity.user = user
@@ -45,7 +45,18 @@ module Omniauthable
       # exists, we assign a temporary email and ask the user to verify it on
       # the next step via Auth::SetupController.show
 
-      user = User.new(user_params_from_auth(auth))
+      strategy          = Devise.omniauth_configs[auth.provider.to_sym].strategy
+      assume_verified   = strategy&.security&.assume_email_is_verified
+      email_is_verified = auth.info.verified || auth.info.verified_email || assume_verified
+      email             = auth.info.verified_email || auth.info.email
+      email             = nil unless email_is_verified
+
+      user = User.find_by(email: email) if email_is_verified
+
+      return user unless user.nil?
+
+      user = User.new(user_params_from_auth(email, auth))
+
       user.account.avatar_remote_url = auth.info.image if auth.info.image =~ /\A#{URI.regexp(%w(http https))}\z/
       user.skip_confirmation!
       user.save!
@@ -54,14 +65,7 @@ module Omniauthable
 
     private
 
-    def user_params_from_auth(auth)
-      strategy          = Devise.omniauth_configs[auth.provider.to_sym].strategy
-      assume_verified   = strategy.try(:security).try(:assume_email_is_verified)
-      email_is_verified = auth.info.verified || auth.info.verified_email || assume_verified
-      email             = auth.info.verified_email || auth.info.email
-      email             = email_is_verified && !User.exists?(email: auth.info.email) && email
-      display_name      = auth.info.full_name || [auth.info.first_name, auth.info.last_name].join(' ')
-
+    def user_params_from_auth(email, auth)
       {
         email: email || "#{TEMP_EMAIL_PREFIX}-#{auth.uid}-#{auth.provider}.com",
         password: Devise.friendly_token[0, 20],
@@ -69,7 +73,7 @@ module Omniauthable
         external: true,
         account_attributes: {
           username: ensure_unique_username(auth.uid),
-          display_name: display_name,
+          display_name: auth.info.full_name || [auth.info.first_name, auth.info.last_name].join(' '),
         },
       }
     end
-- 
cgit 


From 0e5b9e3ba0ddc8061ad57f71924fdbfb981f42a0 Mon Sep 17 00:00:00 2001
From: mayaeh <mayaeh@marimo-net.org>
Date: Tue, 17 Sep 2019 17:06:43 +0900
Subject: Fix the sample account icon of Profile directory to local only
 (#11872)

---
 app/models/tag.rb                    | 2 +-
 app/presenters/instance_presenter.rb | 2 +-
 2 files changed, 2 insertions(+), 2 deletions(-)

(limited to 'app/models')

diff --git a/app/models/tag.rb b/app/models/tag.rb
index 4e77c404d..b52b9bc9f 100644
--- a/app/models/tag.rb
+++ b/app/models/tag.rb
@@ -20,7 +20,7 @@
 class Tag < ApplicationRecord
   has_and_belongs_to_many :statuses
   has_and_belongs_to_many :accounts
-  has_and_belongs_to_many :sample_accounts, -> { searchable.discoverable.popular.limit(3) }, class_name: 'Account'
+  has_and_belongs_to_many :sample_accounts, -> { local.discoverable.popular.limit(3) }, class_name: 'Account'
 
   has_many :featured_tags, dependent: :destroy, inverse_of: :tag
   has_one :account_tag_stat, dependent: :destroy
diff --git a/app/presenters/instance_presenter.rb b/app/presenters/instance_presenter.rb
index f3a73209a..becc92c2d 100644
--- a/app/presenters/instance_presenter.rb
+++ b/app/presenters/instance_presenter.rb
@@ -33,7 +33,7 @@ class InstancePresenter
   end
 
   def sample_accounts
-    Rails.cache.fetch('sample_accounts', expires_in: 12.hours) { Account.discoverable.popular.limit(3) }
+    Rails.cache.fetch('sample_accounts', expires_in: 12.hours) { Account.local.discoverable.popular.limit(3) }
   end
 
   def version_number
-- 
cgit 


From 3919571c3958f7808a7830b7d19d1605fc7c0ef9 Mon Sep 17 00:00:00 2001
From: "han@highemelry" <high.emerly.ytfin@gmail.com>
Date: Wed, 18 Sep 2019 06:20:48 +0900
Subject: Fix audit log error when custom emoji is copied from remote server
 (#11876)

---
 app/models/custom_emoji.rb | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

(limited to 'app/models')

diff --git a/app/models/custom_emoji.rb b/app/models/custom_emoji.rb
index edb1bec75..0dacaf654 100644
--- a/app/models/custom_emoji.rb
+++ b/app/models/custom_emoji.rb
@@ -63,7 +63,7 @@ class CustomEmoji < ApplicationRecord
   def copy!
     copy = self.class.find_or_initialize_by(domain: nil, shortcode: shortcode)
     copy.image = image
-    copy.save!
+    copy.tap(&:save!)
   end
 
   class << self
-- 
cgit 


From a4b60e9ba4874b9ab427bec41d8b2cd252ec4782 Mon Sep 17 00:00:00 2001
From: Eugen Rochko <eugen@zeonfederated.com>
Date: Wed, 18 Sep 2019 02:48:40 +0200
Subject: Fix TOTP codes not being filtered from logs during enabling/disabling
 (#11877)

Not a serious issue because they are meaningless past single use
---
 .../settings/two_factor_authentication/confirmations_controller.rb  | 4 ++--
 app/controllers/settings/two_factor_authentications_controller.rb   | 6 +++---
 app/models/form/two_factor_confirmation.rb                          | 2 +-
 .../settings/two_factor_authentication/confirmations/new.html.haml  | 2 +-
 app/views/settings/two_factor_authentications/show.html.haml        | 2 +-
 .../two_factor_authentication/confirmations_controller_spec.rb      | 6 +++---
 .../settings/two_factor_authentications_controller_spec.rb          | 4 ++--
 7 files changed, 13 insertions(+), 13 deletions(-)

(limited to 'app/models')

diff --git a/app/controllers/settings/two_factor_authentication/confirmations_controller.rb b/app/controllers/settings/two_factor_authentication/confirmations_controller.rb
index 3145e092d..46c90bf74 100644
--- a/app/controllers/settings/two_factor_authentication/confirmations_controller.rb
+++ b/app/controllers/settings/two_factor_authentication/confirmations_controller.rb
@@ -15,7 +15,7 @@ module Settings
       end
 
       def create
-        if current_user.validate_and_consume_otp!(confirmation_params[:code])
+        if current_user.validate_and_consume_otp!(confirmation_params[:otp_attempt])
           flash.now[:notice] = I18n.t('two_factor_authentication.enabled_success')
 
           current_user.otp_required_for_login = true
@@ -33,7 +33,7 @@ module Settings
       private
 
       def confirmation_params
-        params.require(:form_two_factor_confirmation).permit(:code)
+        params.require(:form_two_factor_confirmation).permit(:otp_attempt)
       end
 
       def prepare_two_factor_form
diff --git a/app/controllers/settings/two_factor_authentications_controller.rb b/app/controllers/settings/two_factor_authentications_controller.rb
index 6904076e4..c93b17577 100644
--- a/app/controllers/settings/two_factor_authentications_controller.rb
+++ b/app/controllers/settings/two_factor_authentications_controller.rb
@@ -34,7 +34,7 @@ module Settings
     private
 
     def confirmation_params
-      params.require(:form_two_factor_confirmation).permit(:code)
+      params.require(:form_two_factor_confirmation).permit(:otp_attempt)
     end
 
     def verify_otp_required
@@ -42,8 +42,8 @@ module Settings
     end
 
     def acceptable_code?
-      current_user.validate_and_consume_otp!(confirmation_params[:code]) ||
-        current_user.invalidate_otp_backup_code!(confirmation_params[:code])
+      current_user.validate_and_consume_otp!(confirmation_params[:otp_attempt]) ||
+        current_user.invalidate_otp_backup_code!(confirmation_params[:otp_attempt])
     end
   end
 end
diff --git a/app/models/form/two_factor_confirmation.rb b/app/models/form/two_factor_confirmation.rb
index b8cf76d05..27ada6533 100644
--- a/app/models/form/two_factor_confirmation.rb
+++ b/app/models/form/two_factor_confirmation.rb
@@ -3,5 +3,5 @@
 class Form::TwoFactorConfirmation
   include ActiveModel::Model
 
-  attr_accessor :code
+  attr_accessor :otp_attempt
 end
diff --git a/app/views/settings/two_factor_authentication/confirmations/new.html.haml b/app/views/settings/two_factor_authentication/confirmations/new.html.haml
index e64155299..86cf1f695 100644
--- a/app/views/settings/two_factor_authentication/confirmations/new.html.haml
+++ b/app/views/settings/two_factor_authentication/confirmations/new.html.haml
@@ -12,7 +12,7 @@
       %samp.qr-alternative__code= current_user.otp_secret.scan(/.{4}/).join(' ')
 
   .fields-group
-    = f.input :code, wrapper: :with_label, hint: t('two_factor_authentication.code_hint'), label: t('simple_form.labels.defaults.otp_attempt'), input_html: { :autocomplete => 'off' }, required: true
+    = f.input :otp_attempt, wrapper: :with_label, hint: t('two_factor_authentication.code_hint'), label: t('simple_form.labels.defaults.otp_attempt'), input_html: { :autocomplete => 'off' }, required: true
 
   .actions
     = f.button :button, t('two_factor_authentication.enable'), type: :submit
diff --git a/app/views/settings/two_factor_authentications/show.html.haml b/app/views/settings/two_factor_authentications/show.html.haml
index 259bcd1ef..93509e022 100644
--- a/app/views/settings/two_factor_authentications/show.html.haml
+++ b/app/views/settings/two_factor_authentications/show.html.haml
@@ -10,7 +10,7 @@
   %hr/
 
   = simple_form_for @confirmation, url: settings_two_factor_authentication_path, method: :delete do |f|
-    = f.input :code, wrapper: :with_label, hint: t('two_factor_authentication.code_hint'), label: t('simple_form.labels.defaults.otp_attempt'), input_html: { :autocomplete => 'off' }, required: true
+    = f.input :otp_attempt, wrapper: :with_label, hint: t('two_factor_authentication.code_hint'), label: t('simple_form.labels.defaults.otp_attempt'), input_html: { :autocomplete => 'off' }, required: true
 
     .actions
       = f.button :button, t('two_factor_authentication.disable'), type: :submit
diff --git a/spec/controllers/settings/two_factor_authentication/confirmations_controller_spec.rb b/spec/controllers/settings/two_factor_authentication/confirmations_controller_spec.rb
index 2222a7559..2e5a9325c 100644
--- a/spec/controllers/settings/two_factor_authentication/confirmations_controller_spec.rb
+++ b/spec/controllers/settings/two_factor_authentication/confirmations_controller_spec.rb
@@ -68,7 +68,7 @@ describe Settings::TwoFactorAuthentication::ConfirmationsController do
             true
           end
 
-          post :create, params: { form_two_factor_confirmation: { code: '123456' } }
+          post :create, params: { form_two_factor_confirmation: { otp_attempt: '123456' } }
 
           expect(assigns(:recovery_codes)).to eq otp_backup_codes
           expect(flash[:notice]).to eq 'Two-factor authentication successfully enabled'
@@ -85,7 +85,7 @@ describe Settings::TwoFactorAuthentication::ConfirmationsController do
             false
           end
 
-          post :create, params: { form_two_factor_confirmation: { code: '123456' } }
+          post :create, params: { form_two_factor_confirmation: { otp_attempt: '123456' } }
         end
 
         it 'renders the new view' do
@@ -99,7 +99,7 @@ describe Settings::TwoFactorAuthentication::ConfirmationsController do
 
     context 'when not signed in' do
       it 'redirects if not signed in' do
-        post :create, params: { form_two_factor_confirmation: { code: '123456' } }
+        post :create, params: { form_two_factor_confirmation: { otp_attempt: '123456' } }
         expect(response).to redirect_to('/auth/sign_in')
       end
     end
diff --git a/spec/controllers/settings/two_factor_authentications_controller_spec.rb b/spec/controllers/settings/two_factor_authentications_controller_spec.rb
index f7c628756..922231ded 100644
--- a/spec/controllers/settings/two_factor_authentications_controller_spec.rb
+++ b/spec/controllers/settings/two_factor_authentications_controller_spec.rb
@@ -91,7 +91,7 @@ describe Settings::TwoFactorAuthenticationsController do
           true
         end
 
-        post :destroy, params: { form_two_factor_confirmation: { code: '123456' } }
+        post :destroy, params: { form_two_factor_confirmation: { otp_attempt: '123456' } }
 
         expect(response).to redirect_to(settings_two_factor_authentication_path)
         user.reload
@@ -105,7 +105,7 @@ describe Settings::TwoFactorAuthenticationsController do
           false
         end
 
-        post :destroy, params: { form_two_factor_confirmation: { code: '057772' } }
+        post :destroy, params: { form_two_factor_confirmation: { otp_attempt: '057772' } }
 
         user.reload
         expect(user.otp_required_for_login).to eq(true)
-- 
cgit 


From e1066cd4319a220d5be16e51ffaf5236a2f6e866 Mon Sep 17 00:00:00 2001
From: Eugen Rochko <eugen@zeonfederated.com>
Date: Wed, 18 Sep 2019 16:37:27 +0200
Subject: Add password challenge to 2FA settings, e-mail notifications (#11878)

Fix #3961
---
 .../admin/two_factor_authentications_controller.rb |   1 +
 app/controllers/auth/challenges_controller.rb      |  22 ++++
 app/controllers/auth/sessions_controller.rb        |   1 +
 app/controllers/concerns/challengable_concern.rb   |  65 ++++++++++++
 .../confirmations_controller.rb                    |   5 +
 .../recovery_codes_controller.rb                   |   6 ++
 .../two_factor_authentications_controller.rb       |   4 +
 app/javascript/styles/mastodon/admin.scss          |  43 ++++----
 app/javascript/styles/mastodon/forms.scss          |   4 +
 app/mailers/user_mailer.rb                         |  33 ++++++
 app/models/form/challenge.rb                       |   8 ++
 app/models/user.rb                                 |   9 +-
 app/views/auth/challenges/new.html.haml            |  15 +++
 app/views/auth/shared/_links.html.haml             |   2 +-
 .../two_factor_authentications/show.html.haml      |  38 +++----
 .../user_mailer/two_factor_disabled.html.haml      |  43 ++++++++
 app/views/user_mailer/two_factor_disabled.text.erb |   7 ++
 app/views/user_mailer/two_factor_enabled.html.haml |  43 ++++++++
 app/views/user_mailer/two_factor_enabled.text.erb  |   7 ++
 .../two_factor_recovery_codes_changed.html.haml    |  43 ++++++++
 .../two_factor_recovery_codes_changed.text.erb     |   7 ++
 config/locales/devise.en.yml                       |  12 +++
 config/locales/en.yml                              |   5 +
 config/locales/simple_form.en.yml                  |   2 +
 config/routes.rb                                   |   1 +
 .../controllers/auth/challenges_controller_spec.rb |  46 +++++++++
 spec/controllers/auth/sessions_controller_spec.rb  |   2 +-
 .../concerns/challengable_concern_spec.rb          | 114 +++++++++++++++++++++
 .../confirmations_controller_spec.rb               |  10 +-
 .../recovery_codes_controller_spec.rb              |   2 +-
 .../two_factor_authentications_controller_spec.rb  |   2 +-
 spec/mailers/previews/user_mailer_preview.rb       |  15 +++
 32 files changed, 567 insertions(+), 50 deletions(-)
 create mode 100644 app/controllers/auth/challenges_controller.rb
 create mode 100644 app/controllers/concerns/challengable_concern.rb
 create mode 100644 app/models/form/challenge.rb
 create mode 100644 app/views/auth/challenges/new.html.haml
 create mode 100644 app/views/user_mailer/two_factor_disabled.html.haml
 create mode 100644 app/views/user_mailer/two_factor_disabled.text.erb
 create mode 100644 app/views/user_mailer/two_factor_enabled.html.haml
 create mode 100644 app/views/user_mailer/two_factor_enabled.text.erb
 create mode 100644 app/views/user_mailer/two_factor_recovery_codes_changed.html.haml
 create mode 100644 app/views/user_mailer/two_factor_recovery_codes_changed.text.erb
 create mode 100644 spec/controllers/auth/challenges_controller_spec.rb
 create mode 100644 spec/controllers/concerns/challengable_concern_spec.rb

(limited to 'app/models')

diff --git a/app/controllers/admin/two_factor_authentications_controller.rb b/app/controllers/admin/two_factor_authentications_controller.rb
index 2577a4b17..0652c3a7a 100644
--- a/app/controllers/admin/two_factor_authentications_controller.rb
+++ b/app/controllers/admin/two_factor_authentications_controller.rb
@@ -8,6 +8,7 @@ module Admin
       authorize @user, :disable_2fa?
       @user.disable_two_factor!
       log_action :disable_2fa, @user
+      UserMailer.two_factor_disabled(@user).deliver_later!
       redirect_to admin_accounts_path
     end
 
diff --git a/app/controllers/auth/challenges_controller.rb b/app/controllers/auth/challenges_controller.rb
new file mode 100644
index 000000000..060944240
--- /dev/null
+++ b/app/controllers/auth/challenges_controller.rb
@@ -0,0 +1,22 @@
+# frozen_string_literal: true
+
+class Auth::ChallengesController < ApplicationController
+  include ChallengableConcern
+
+  layout 'auth'
+
+  before_action :authenticate_user!
+
+  skip_before_action :require_functional!
+
+  def create
+    if challenge_passed?
+      session[:challenge_passed_at] = Time.now.utc
+      redirect_to challenge_params[:return_to]
+    else
+      @challenge = Form::Challenge.new(return_to: challenge_params[:return_to])
+      flash.now[:alert] = I18n.t('challenge.invalid_password')
+      render_challenge
+    end
+  end
+end
diff --git a/app/controllers/auth/sessions_controller.rb b/app/controllers/auth/sessions_controller.rb
index 3e93b2e68..b3113bbef 100644
--- a/app/controllers/auth/sessions_controller.rb
+++ b/app/controllers/auth/sessions_controller.rb
@@ -42,6 +42,7 @@ class Auth::SessionsController < Devise::SessionsController
   def destroy
     tmp_stored_location = stored_location_for(:user)
     super
+    session.delete(:challenge_passed_at)
     flash.delete(:notice)
     store_location_for(:user, tmp_stored_location) if continue_after?
   end
diff --git a/app/controllers/concerns/challengable_concern.rb b/app/controllers/concerns/challengable_concern.rb
new file mode 100644
index 000000000..b29d90b3c
--- /dev/null
+++ b/app/controllers/concerns/challengable_concern.rb
@@ -0,0 +1,65 @@
+# frozen_string_literal: true
+
+# This concern is inspired by "sudo mode" on GitHub. It
+# is a way to re-authenticate a user before allowing them
+# to see or perform an action.
+#
+# Add `before_action :require_challenge!` to actions you
+# want to protect.
+#
+# The user will be shown a page to enter the challenge (which
+# is either the password, or just the username when no
+# password exists). Upon passing, there is a grace period
+# during which no challenge will be asked from the user.
+#
+# Accessing challenge-protected resources during the grace
+# period will refresh the grace period.
+module ChallengableConcern
+  extend ActiveSupport::Concern
+
+  CHALLENGE_TIMEOUT = 1.hour.freeze
+
+  def require_challenge!
+    return if skip_challenge?
+
+    if challenge_passed_recently?
+      session[:challenge_passed_at] = Time.now.utc
+      return
+    end
+
+    @challenge = Form::Challenge.new(return_to: request.url)
+
+    if params.key?(:form_challenge)
+      if challenge_passed?
+        session[:challenge_passed_at] = Time.now.utc
+        return
+      else
+        flash.now[:alert] = I18n.t('challenge.invalid_password')
+        render_challenge
+      end
+    else
+      render_challenge
+    end
+  end
+
+  def render_challenge
+    @body_classes = 'lighter'
+    render template: 'auth/challenges/new', layout: 'auth'
+  end
+
+  def challenge_passed?
+    current_user.valid_password?(challenge_params[:current_password])
+  end
+
+  def skip_challenge?
+    current_user.encrypted_password.blank?
+  end
+
+  def challenge_passed_recently?
+    session[:challenge_passed_at].present? && session[:challenge_passed_at] >= CHALLENGE_TIMEOUT.ago
+  end
+
+  def challenge_params
+    params.require(:form_challenge).permit(:current_password, :return_to)
+  end
+end
diff --git a/app/controllers/settings/two_factor_authentication/confirmations_controller.rb b/app/controllers/settings/two_factor_authentication/confirmations_controller.rb
index 46c90bf74..ef4df3339 100644
--- a/app/controllers/settings/two_factor_authentication/confirmations_controller.rb
+++ b/app/controllers/settings/two_factor_authentication/confirmations_controller.rb
@@ -3,9 +3,12 @@
 module Settings
   module TwoFactorAuthentication
     class ConfirmationsController < BaseController
+      include ChallengableConcern
+
       layout 'admin'
 
       before_action :authenticate_user!
+      before_action :require_challenge!
       before_action :ensure_otp_secret
 
       skip_before_action :require_functional!
@@ -22,6 +25,8 @@ module Settings
           @recovery_codes = current_user.generate_otp_backup_codes!
           current_user.save!
 
+          UserMailer.two_factor_enabled(current_user).deliver_later!
+
           render 'settings/two_factor_authentication/recovery_codes/index'
         else
           flash.now[:alert] = I18n.t('two_factor_authentication.wrong_code')
diff --git a/app/controllers/settings/two_factor_authentication/recovery_codes_controller.rb b/app/controllers/settings/two_factor_authentication/recovery_codes_controller.rb
index 09a759860..0c4f5bff7 100644
--- a/app/controllers/settings/two_factor_authentication/recovery_codes_controller.rb
+++ b/app/controllers/settings/two_factor_authentication/recovery_codes_controller.rb
@@ -3,16 +3,22 @@
 module Settings
   module TwoFactorAuthentication
     class RecoveryCodesController < BaseController
+      include ChallengableConcern
+
       layout 'admin'
 
       before_action :authenticate_user!
+      before_action :require_challenge!, on: :create
 
       skip_before_action :require_functional!
 
       def create
         @recovery_codes = current_user.generate_otp_backup_codes!
         current_user.save!
+
+        UserMailer.two_factor_recovery_codes_changed(current_user).deliver_later!
         flash.now[:notice] = I18n.t('two_factor_authentication.recovery_codes_regenerated')
+
         render :index
       end
     end
diff --git a/app/controllers/settings/two_factor_authentications_controller.rb b/app/controllers/settings/two_factor_authentications_controller.rb
index c93b17577..9118a7933 100644
--- a/app/controllers/settings/two_factor_authentications_controller.rb
+++ b/app/controllers/settings/two_factor_authentications_controller.rb
@@ -2,10 +2,13 @@
 
 module Settings
   class TwoFactorAuthenticationsController < BaseController
+    include ChallengableConcern
+
     layout 'admin'
 
     before_action :authenticate_user!
     before_action :verify_otp_required, only: [:create]
+    before_action :require_challenge!, only: [:create]
 
     skip_before_action :require_functional!
 
@@ -23,6 +26,7 @@ module Settings
       if acceptable_code?
         current_user.otp_required_for_login = false
         current_user.save!
+        UserMailer.two_factor_disabled(current_user).deliver_later!
         redirect_to settings_two_factor_authentication_path
       else
         flash.now[:alert] = I18n.t('two_factor_authentication.wrong_code')
diff --git a/app/javascript/styles/mastodon/admin.scss b/app/javascript/styles/mastodon/admin.scss
index 5d4fe4ef8..074eee2cd 100644
--- a/app/javascript/styles/mastodon/admin.scss
+++ b/app/javascript/styles/mastodon/admin.scss
@@ -233,32 +233,35 @@ hr.spacer {
   height: 1px;
 }
 
-.muted-hint {
-  color: $darker-text-color;
+body,
+.admin-wrapper .content {
+  .muted-hint {
+    color: $darker-text-color;
 
-  a {
-    color: $highlight-text-color;
+    a {
+      color: $highlight-text-color;
+    }
   }
-}
 
-.positive-hint {
-  color: $valid-value-color;
-  font-weight: 500;
-}
+  .positive-hint {
+    color: $valid-value-color;
+    font-weight: 500;
+  }
 
-.negative-hint {
-  color: $error-value-color;
-  font-weight: 500;
-}
+  .negative-hint {
+    color: $error-value-color;
+    font-weight: 500;
+  }
 
-.neutral-hint {
-  color: $dark-text-color;
-  font-weight: 500;
-}
+  .neutral-hint {
+    color: $dark-text-color;
+    font-weight: 500;
+  }
 
-.warning-hint {
-  color: $gold-star;
-  font-weight: 500;
+  .warning-hint {
+    color: $gold-star;
+    font-weight: 500;
+  }
 }
 
 .filters {
diff --git a/app/javascript/styles/mastodon/forms.scss b/app/javascript/styles/mastodon/forms.scss
index 16352340b..80ef8797d 100644
--- a/app/javascript/styles/mastodon/forms.scss
+++ b/app/javascript/styles/mastodon/forms.scss
@@ -254,6 +254,10 @@ code {
       &-6 {
         max-width: 50%;
       }
+
+      .actions {
+        margin-top: 27px;
+      }
     }
 
     .fields-group:last-child,
diff --git a/app/mailers/user_mailer.rb b/app/mailers/user_mailer.rb
index b41004acc..6b81f6873 100644
--- a/app/mailers/user_mailer.rb
+++ b/app/mailers/user_mailer.rb
@@ -57,6 +57,39 @@ class UserMailer < Devise::Mailer
     end
   end
 
+  def two_factor_enabled(user, **)
+    @resource = user
+    @instance = Rails.configuration.x.local_domain
+
+    return if @resource.disabled?
+
+    I18n.with_locale(@resource.locale || I18n.default_locale) do
+      mail to: @resource.email, subject: I18n.t('devise.mailer.two_factor_enabled.subject')
+    end
+  end
+
+  def two_factor_disabled(user, **)
+    @resource = user
+    @instance = Rails.configuration.x.local_domain
+
+    return if @resource.disabled?
+
+    I18n.with_locale(@resource.locale || I18n.default_locale) do
+      mail to: @resource.email, subject: I18n.t('devise.mailer.two_factor_disabled.subject')
+    end
+  end
+
+  def two_factor_recovery_codes_changed(user, **)
+    @resource = user
+    @instance = Rails.configuration.x.local_domain
+
+    return if @resource.disabled?
+
+    I18n.with_locale(@resource.locale || I18n.default_locale) do
+      mail to: @resource.email, subject: I18n.t('devise.mailer.two_factor_recovery_codes_changed.subject')
+    end
+  end
+
   def welcome(user)
     @resource = user
     @instance = Rails.configuration.x.local_domain
diff --git a/app/models/form/challenge.rb b/app/models/form/challenge.rb
new file mode 100644
index 000000000..40c99649c
--- /dev/null
+++ b/app/models/form/challenge.rb
@@ -0,0 +1,8 @@
+# frozen_string_literal: true
+
+class Form::Challenge
+  include ActiveModel::Model
+
+  attr_accessor :current_password, :current_username,
+                :return_to
+end
diff --git a/app/models/user.rb b/app/models/user.rb
index 78b82a68f..b48455802 100644
--- a/app/models/user.rb
+++ b/app/models/user.rb
@@ -264,17 +264,20 @@ class User < ApplicationRecord
   end
 
   def password_required?
-    return false if Devise.pam_authentication || Devise.ldap_authentication
+    return false if external?
+
     super
   end
 
   def send_reset_password_instructions
-    return false if encrypted_password.blank? && (Devise.pam_authentication || Devise.ldap_authentication)
+    return false if encrypted_password.blank?
+
     super
   end
 
   def reset_password!(new_password, new_password_confirmation)
-    return false if encrypted_password.blank? && (Devise.pam_authentication || Devise.ldap_authentication)
+    return false if encrypted_password.blank?
+
     super
   end
 
diff --git a/app/views/auth/challenges/new.html.haml b/app/views/auth/challenges/new.html.haml
new file mode 100644
index 000000000..9aef2c35d
--- /dev/null
+++ b/app/views/auth/challenges/new.html.haml
@@ -0,0 +1,15 @@
+- content_for :page_title do
+  = t('challenge.prompt')
+
+= simple_form_for @challenge, url: request.get? ? auth_challenge_path : '' do |f|
+  = f.input :return_to, as: :hidden
+
+  .field-group
+    = f.input :current_password, wrapper: :with_block_label, input_html: { :autocomplete => 'off', :autofocus => true }, label: t('challenge.prompt'), required: true
+
+  .actions
+    = f.button :button, t('challenge.confirm'), type: :submit
+
+  %p.hint.subtle-hint= t('challenge.hint_html')
+
+.form-footer= render 'auth/shared/links'
diff --git a/app/views/auth/shared/_links.html.haml b/app/views/auth/shared/_links.html.haml
index e6c3f7cca..66ed5b93f 100644
--- a/app/views/auth/shared/_links.html.haml
+++ b/app/views/auth/shared/_links.html.haml
@@ -11,7 +11,7 @@
     - if controller_name != 'passwords' && controller_name != 'registrations'
       %li= link_to t('auth.forgot_password'), new_user_password_path
 
-  - if controller_name != 'confirmations'
+  - if controller_name != 'confirmations' && (!user_signed_in? || !current_user.confirmed? || current_user.unconfirmed_email.present?)
     %li= link_to t('auth.didnt_get_confirmation'), new_user_confirmation_path
 
   - if user_signed_in? && controller_name != 'setup'
diff --git a/app/views/settings/two_factor_authentications/show.html.haml b/app/views/settings/two_factor_authentications/show.html.haml
index 93509e022..f1eecd000 100644
--- a/app/views/settings/two_factor_authentications/show.html.haml
+++ b/app/views/settings/two_factor_authentications/show.html.haml
@@ -2,33 +2,35 @@
   = t('settings.two_factor_authentication')
 
 - if current_user.otp_required_for_login
-  %p.positive-hint
-    = fa_icon 'check'
-    = ' '
-    = t 'two_factor_authentication.enabled'
+  %p.hint
+    %span.positive-hint
+      = fa_icon 'check'
+      = ' '
+      = t 'two_factor_authentication.enabled'
 
-  %hr/
+  %hr.spacer/
 
   = simple_form_for @confirmation, url: settings_two_factor_authentication_path, method: :delete do |f|
-    = f.input :otp_attempt, wrapper: :with_label, hint: t('two_factor_authentication.code_hint'), label: t('simple_form.labels.defaults.otp_attempt'), input_html: { :autocomplete => 'off' }, required: true
+    .fields-group
+      = f.input :otp_attempt, wrapper: :with_block_label, hint: t('two_factor_authentication.code_hint'), label: t('simple_form.labels.defaults.otp_attempt'), input_html: { :autocomplete => 'off' }, required: true
 
     .actions
-      = f.button :button, t('two_factor_authentication.disable'), type: :submit
+      = f.button :button, t('two_factor_authentication.disable'), type: :submit, class: 'negative'
 
-  %hr/
+  %hr.spacer/
 
-  %h6= t('two_factor_authentication.recovery_codes')
-  %p.muted-hint
-    = t('two_factor_authentication.lost_recovery_codes')
-    = link_to t('two_factor_authentication.generate_recovery_codes'),
-      settings_two_factor_authentication_recovery_codes_path,
-      data: { method: :post }
+  %h3= t('two_factor_authentication.recovery_codes')
+  %p.muted-hint= t('two_factor_authentication.lost_recovery_codes')
+
+  %hr.spacer/
+
+  .simple_form
+    = link_to t('two_factor_authentication.generate_recovery_codes'), settings_two_factor_authentication_recovery_codes_path, data: { method: :post }, class: 'block-button'
 
 - else
   .simple_form
     %p.hint= t('two_factor_authentication.description_html')
 
-    = link_to t('two_factor_authentication.setup'),
-      settings_two_factor_authentication_path,
-      data: { method: :post },
-      class: 'block-button'
+    %hr.spacer/
+
+    = link_to t('two_factor_authentication.setup'), settings_two_factor_authentication_path, data: { method: :post }, class: 'block-button'
diff --git a/app/views/user_mailer/two_factor_disabled.html.haml b/app/views/user_mailer/two_factor_disabled.html.haml
new file mode 100644
index 000000000..651c6f940
--- /dev/null
+++ b/app/views/user_mailer/two_factor_disabled.html.haml
@@ -0,0 +1,43 @@
+%table.email-table{ cellspacing: 0, cellpadding: 0 }
+  %tbody
+    %tr
+      %td.email-body
+        .email-container
+          %table.content-section{ cellspacing: 0, cellpadding: 0 }
+            %tbody
+              %tr
+                %td.content-cell.hero
+                  .email-row
+                    .col-6
+                      %table.column{ cellspacing: 0, cellpadding: 0 }
+                        %tbody
+                          %tr
+                            %td.column-cell.text-center.padded
+                              %table.hero-icon.alert-icon{ align: 'center', cellspacing: 0, cellpadding: 0 }
+                                %tbody
+                                  %tr
+                                    %td
+                                      = image_tag full_pack_url('media/images/mailer/icon_lock_open.png'), alt: ''
+
+                              %h1= t 'devise.mailer.two_factor_disabled.title'
+                              %p.lead= t 'devise.mailer.two_factor_disabled.explanation'
+
+%table.email-table{ cellspacing: 0, cellpadding: 0 }
+  %tbody
+    %tr
+      %td.email-body
+        .email-container
+          %table.content-section{ cellspacing: 0, cellpadding: 0 }
+            %tbody
+              %tr
+                %td.content-cell.content-start
+                  %table.column{ cellspacing: 0, cellpadding: 0 }
+                    %tbody
+                      %tr
+                        %td.column-cell.button-cell
+                          %table.button{ align: 'center', cellspacing: 0, cellpadding: 0 }
+                            %tbody
+                              %tr
+                                %td.button-primary
+                                  = link_to edit_user_registration_url do
+                                    %span= t('settings.account_settings')
diff --git a/app/views/user_mailer/two_factor_disabled.text.erb b/app/views/user_mailer/two_factor_disabled.text.erb
new file mode 100644
index 000000000..73be1ddc2
--- /dev/null
+++ b/app/views/user_mailer/two_factor_disabled.text.erb
@@ -0,0 +1,7 @@
+<%= t 'devise.mailer.two_factor_disabled.title' %>
+
+===
+
+<%= t 'devise.mailer.two_factor_disabled.explanation' %>
+
+=> <%= edit_user_registration_url %>
diff --git a/app/views/user_mailer/two_factor_enabled.html.haml b/app/views/user_mailer/two_factor_enabled.html.haml
new file mode 100644
index 000000000..fc31bd979
--- /dev/null
+++ b/app/views/user_mailer/two_factor_enabled.html.haml
@@ -0,0 +1,43 @@
+%table.email-table{ cellspacing: 0, cellpadding: 0 }
+  %tbody
+    %tr
+      %td.email-body
+        .email-container
+          %table.content-section{ cellspacing: 0, cellpadding: 0 }
+            %tbody
+              %tr
+                %td.content-cell.hero
+                  .email-row
+                    .col-6
+                      %table.column{ cellspacing: 0, cellpadding: 0 }
+                        %tbody
+                          %tr
+                            %td.column-cell.text-center.padded
+                              %table.hero-icon{ align: 'center', cellspacing: 0, cellpadding: 0 }
+                                %tbody
+                                  %tr
+                                    %td
+                                      = image_tag full_pack_url('media/images/mailer/icon_lock_open.png'), alt: ''
+
+                              %h1= t 'devise.mailer.two_factor_enabled.title'
+                              %p.lead= t 'devise.mailer.two_factor_enabled.explanation'
+
+%table.email-table{ cellspacing: 0, cellpadding: 0 }
+  %tbody
+    %tr
+      %td.email-body
+        .email-container
+          %table.content-section{ cellspacing: 0, cellpadding: 0 }
+            %tbody
+              %tr
+                %td.content-cell.content-start
+                  %table.column{ cellspacing: 0, cellpadding: 0 }
+                    %tbody
+                      %tr
+                        %td.column-cell.button-cell
+                          %table.button{ align: 'center', cellspacing: 0, cellpadding: 0 }
+                            %tbody
+                              %tr
+                                %td.button-primary
+                                  = link_to edit_user_registration_url do
+                                    %span= t('settings.account_settings')
diff --git a/app/views/user_mailer/two_factor_enabled.text.erb b/app/views/user_mailer/two_factor_enabled.text.erb
new file mode 100644
index 000000000..4319dddbf
--- /dev/null
+++ b/app/views/user_mailer/two_factor_enabled.text.erb
@@ -0,0 +1,7 @@
+<%= t 'devise.mailer.two_factor_enabled.title' %>
+
+===
+
+<%= t 'devise.mailer.two_factor_enabled.explanation' %>
+
+=> <%= edit_user_registration_url %>
diff --git a/app/views/user_mailer/two_factor_recovery_codes_changed.html.haml b/app/views/user_mailer/two_factor_recovery_codes_changed.html.haml
new file mode 100644
index 000000000..833708868
--- /dev/null
+++ b/app/views/user_mailer/two_factor_recovery_codes_changed.html.haml
@@ -0,0 +1,43 @@
+%table.email-table{ cellspacing: 0, cellpadding: 0 }
+  %tbody
+    %tr
+      %td.email-body
+        .email-container
+          %table.content-section{ cellspacing: 0, cellpadding: 0 }
+            %tbody
+              %tr
+                %td.content-cell.hero
+                  .email-row
+                    .col-6
+                      %table.column{ cellspacing: 0, cellpadding: 0 }
+                        %tbody
+                          %tr
+                            %td.column-cell.text-center.padded
+                              %table.hero-icon.alert-icon{ align: 'center', cellspacing: 0, cellpadding: 0 }
+                                %tbody
+                                  %tr
+                                    %td
+                                      = image_tag full_pack_url('media/images/mailer/icon_lock_open.png'), alt: ''
+
+                              %h1= t 'devise.mailer.two_factor_recovery_codes_changed.title'
+                              %p.lead= t 'devise.mailer.two_factor_recovery_codes_changed.explanation'
+
+%table.email-table{ cellspacing: 0, cellpadding: 0 }
+  %tbody
+    %tr
+      %td.email-body
+        .email-container
+          %table.content-section{ cellspacing: 0, cellpadding: 0 }
+            %tbody
+              %tr
+                %td.content-cell.content-start
+                  %table.column{ cellspacing: 0, cellpadding: 0 }
+                    %tbody
+                      %tr
+                        %td.column-cell.button-cell
+                          %table.button{ align: 'center', cellspacing: 0, cellpadding: 0 }
+                            %tbody
+                              %tr
+                                %td.button-primary
+                                  = link_to edit_user_registration_url do
+                                    %span= t('settings.account_settings')
diff --git a/app/views/user_mailer/two_factor_recovery_codes_changed.text.erb b/app/views/user_mailer/two_factor_recovery_codes_changed.text.erb
new file mode 100644
index 000000000..6ed12fc08
--- /dev/null
+++ b/app/views/user_mailer/two_factor_recovery_codes_changed.text.erb
@@ -0,0 +1,7 @@
+<%= t 'devise.mailer.two_factor_recovery_codes_changed.title' %>
+
+===
+
+<%= t 'devise.mailer.two_factor_recovery_codes_changed.explanation' %>
+
+=> <%= edit_user_registration_url %>
diff --git a/config/locales/devise.en.yml b/config/locales/devise.en.yml
index 5defa6624..726d2426a 100644
--- a/config/locales/devise.en.yml
+++ b/config/locales/devise.en.yml
@@ -46,6 +46,18 @@ en:
         extra: If you didn't request this, please ignore this email. Your password won't change until you access the link above and create a new one.
         subject: 'Mastodon: Reset password instructions'
         title: Password reset
+      two_factor_disabled:
+        explanation: Two-factor authentication for your account has been disabled. Login is now possible using only e-mail address and password.
+        subject: 'Mastodon: Two-factor authentication disabled'
+        title: 2FA disabled
+      two_factor_enabled:
+        explanation: Two-factor authentication has been enabled for your account. A token generated by the paired TOTP app will be required for login.
+        subject: 'Mastodon: Two-factor authentication enabled'
+        title: 2FA enabled
+      two_factor_recovery_codes_changed:
+        explanation: The previous recovery codes have been invalidated and new ones generated.
+        subject: 'Mastodon: Two-factor recovery codes re-generated'
+        title: 2FA recovery codes changed
       unlock_instructions:
         subject: 'Mastodon: Unlock instructions'
     omniauth_callbacks:
diff --git a/config/locales/en.yml b/config/locales/en.yml
index f05fdd48b..da06b0e51 100644
--- a/config/locales/en.yml
+++ b/config/locales/en.yml
@@ -621,6 +621,11 @@ en:
       return: Show the user's profile
       web: Go to web
     title: Follow %{acct}
+  challenge:
+    confirm: Continue
+    hint_html: "<strong>Tip:</strong> We won't ask you for your password again for the next hour."
+    invalid_password: Invalid password
+    prompt: Confirm password to continue
   datetime:
     distance_in_words:
       about_x_hours: "%{count}h"
diff --git a/config/locales/simple_form.en.yml b/config/locales/simple_form.en.yml
index c542377a9..c9ffcfc13 100644
--- a/config/locales/simple_form.en.yml
+++ b/config/locales/simple_form.en.yml
@@ -43,6 +43,8 @@ en:
         domain: This domain will be able to fetch data from this server and incoming data from it will be processed and stored
       featured_tag:
         name: 'You might want to use one of these:'
+      form_challenge:
+        current_password: You are entering a secure area
       imports:
         data: CSV file exported from another Mastodon server
       invite_request:
diff --git a/config/routes.rb b/config/routes.rb
index a4dee2842..9ad1ea65d 100644
--- a/config/routes.rb
+++ b/config/routes.rb
@@ -41,6 +41,7 @@ Rails.application.routes.draw do
 
     namespace :auth do
       resource :setup, only: [:show, :update], controller: :setup
+      resource :challenge, only: [:create], controller: :challenges
     end
   end
 
diff --git a/spec/controllers/auth/challenges_controller_spec.rb b/spec/controllers/auth/challenges_controller_spec.rb
new file mode 100644
index 000000000..2a6ca301e
--- /dev/null
+++ b/spec/controllers/auth/challenges_controller_spec.rb
@@ -0,0 +1,46 @@
+# frozen_string_literal: true
+
+require 'rails_helper'
+
+describe Auth::ChallengesController, type: :controller do
+  render_views
+
+  let(:password) { 'foobar12345' }
+  let(:user) { Fabricate(:user, password: password) }
+
+  before do
+    sign_in user
+  end
+
+  describe 'POST #create' do
+    let(:return_to) { edit_user_registration_path }
+
+    context 'with correct password' do
+      before { post :create, params: { form_challenge: { return_to: return_to, current_password: password } } }
+
+      it 'redirects back' do
+        expect(response).to redirect_to(return_to)
+      end
+
+      it 'sets session' do
+        expect(session[:challenge_passed_at]).to_not be_nil
+      end
+    end
+
+    context 'with incorrect password' do
+      before { post :create, params: { form_challenge: { return_to: return_to, current_password: 'hhfggjjd562' } } }
+
+      it 'renders challenge' do
+        expect(response).to render_template('auth/challenges/new')
+      end
+
+      it 'displays error' do
+        expect(response.body).to include 'Invalid password'
+      end
+
+      it 'does not set session' do
+        expect(session[:challenge_passed_at]).to be_nil
+      end
+    end
+  end
+end
diff --git a/spec/controllers/auth/sessions_controller_spec.rb b/spec/controllers/auth/sessions_controller_spec.rb
index 7ed5edde0..1950c173a 100644
--- a/spec/controllers/auth/sessions_controller_spec.rb
+++ b/spec/controllers/auth/sessions_controller_spec.rb
@@ -80,7 +80,7 @@ RSpec.describe Auth::SessionsController, type: :controller do
         let(:user) do
           account = Fabricate.build(:account, username: 'pam_user1')
           account.save!(validate: false)
-          user = Fabricate(:user, email: 'pam@example.com', password: nil, account: account)
+          user = Fabricate(:user, email: 'pam@example.com', password: nil, account: account, external: true)
           user
         end
 
diff --git a/spec/controllers/concerns/challengable_concern_spec.rb b/spec/controllers/concerns/challengable_concern_spec.rb
new file mode 100644
index 000000000..4db3b740d
--- /dev/null
+++ b/spec/controllers/concerns/challengable_concern_spec.rb
@@ -0,0 +1,114 @@
+# frozen_string_literal: true
+
+require 'rails_helper'
+
+RSpec.describe ChallengableConcern, type: :controller do
+  controller(ApplicationController) do
+    include ChallengableConcern
+
+    before_action :require_challenge!
+
+    def foo
+      render plain: 'foo'
+    end
+
+    def bar
+      render plain: 'bar'
+    end
+  end
+
+  before do
+    routes.draw do
+      get  'foo' => 'anonymous#foo'
+      post 'bar' => 'anonymous#bar'
+    end
+  end
+
+  context 'with a no-password user' do
+    let(:user) { Fabricate(:user, external: true, password: nil) }
+
+    before do
+      sign_in user
+    end
+
+    context 'for GET requests' do
+      before { get :foo }
+
+      it 'does not ask for password' do
+        expect(response.body).to eq 'foo'
+      end
+    end
+
+    context 'for POST requests' do
+      before { post :bar }
+
+      it 'does not ask for password' do
+        expect(response.body).to eq 'bar'
+      end
+    end
+  end
+
+  context 'with recent challenge in session' do
+    let(:password) { 'foobar12345' }
+    let(:user) { Fabricate(:user, password: password) }
+
+    before do
+      sign_in user
+    end
+
+    context 'for GET requests' do
+      before { get :foo, session: { challenge_passed_at: Time.now.utc } }
+
+      it 'does not ask for password' do
+        expect(response.body).to eq 'foo'
+      end
+    end
+
+    context 'for POST requests' do
+      before { post :bar, session: { challenge_passed_at: Time.now.utc } }
+
+      it 'does not ask for password' do
+        expect(response.body).to eq 'bar'
+      end
+    end
+  end
+
+  context 'with a password user' do
+    let(:password) { 'foobar12345' }
+    let(:user) { Fabricate(:user, password: password) }
+
+    before do
+      sign_in user
+    end
+
+    context 'for GET requests' do
+      before { get :foo }
+
+      it 'renders challenge' do
+        expect(response).to render_template('auth/challenges/new')
+      end
+
+      # See Auth::ChallengesControllerSpec
+    end
+
+    context 'for POST requests' do
+      before { post :bar }
+
+      it 'renders challenge' do
+        expect(response).to render_template('auth/challenges/new')
+      end
+
+      it 'accepts correct password' do
+        post :bar, params: { form_challenge: { current_password: password } }
+        expect(response.body).to eq 'bar'
+        expect(session[:challenge_passed_at]).to_not be_nil
+      end
+
+      it 'rejects wrong password' do
+        post :bar, params: { form_challenge: { current_password: 'dddfff888123' } }
+        expect(response.body).to render_template('auth/challenges/new')
+        expect(session[:challenge_passed_at]).to be_nil
+      end
+    end
+  end
+end
diff --git a/spec/controllers/settings/two_factor_authentication/confirmations_controller_spec.rb b/spec/controllers/settings/two_factor_authentication/confirmations_controller_spec.rb
index 2e5a9325c..336f13127 100644
--- a/spec/controllers/settings/two_factor_authentication/confirmations_controller_spec.rb
+++ b/spec/controllers/settings/two_factor_authentication/confirmations_controller_spec.rb
@@ -24,7 +24,7 @@ describe Settings::TwoFactorAuthentication::ConfirmationsController do
     context 'when signed in' do
       subject do
         sign_in user, scope: :user
-        get :new
+        get :new, session: { challenge_passed_at: Time.now.utc }
       end
 
       include_examples 'renders :new'
@@ -37,7 +37,7 @@ describe Settings::TwoFactorAuthentication::ConfirmationsController do
 
     it 'redirects if user do not have otp_secret' do
       sign_in user_without_otp_secret, scope: :user
-      get :new
+      get :new, session: { challenge_passed_at: Time.now.utc }
       expect(response).to redirect_to('/settings/two_factor_authentication')
     end
   end
@@ -50,7 +50,7 @@ describe Settings::TwoFactorAuthentication::ConfirmationsController do
 
       describe 'when form_two_factor_confirmation parameter is not provided' do
         it 'raises ActionController::ParameterMissing' do
-          post :create, params: {}
+          post :create, params: {}, session: { challenge_passed_at: Time.now.utc }
           expect(response).to have_http_status(400)
         end
       end
@@ -68,7 +68,7 @@ describe Settings::TwoFactorAuthentication::ConfirmationsController do
             true
           end
 
-          post :create, params: { form_two_factor_confirmation: { otp_attempt: '123456' } }
+          post :create, params: { form_two_factor_confirmation: { otp_attempt: '123456' } }, session: { challenge_passed_at: Time.now.utc }
 
           expect(assigns(:recovery_codes)).to eq otp_backup_codes
           expect(flash[:notice]).to eq 'Two-factor authentication successfully enabled'
@@ -85,7 +85,7 @@ describe Settings::TwoFactorAuthentication::ConfirmationsController do
             false
           end
 
-          post :create, params: { form_two_factor_confirmation: { otp_attempt: '123456' } }
+          post :create, params: { form_two_factor_confirmation: { otp_attempt: '123456' } }, session: { challenge_passed_at: Time.now.utc }
         end
 
         it 'renders the new view' do
diff --git a/spec/controllers/settings/two_factor_authentication/recovery_codes_controller_spec.rb b/spec/controllers/settings/two_factor_authentication/recovery_codes_controller_spec.rb
index c04760e53..630cec428 100644
--- a/spec/controllers/settings/two_factor_authentication/recovery_codes_controller_spec.rb
+++ b/spec/controllers/settings/two_factor_authentication/recovery_codes_controller_spec.rb
@@ -15,7 +15,7 @@ describe Settings::TwoFactorAuthentication::RecoveryCodesController do
       end
 
       sign_in user, scope: :user
-      post :create
+      post :create, session: { challenge_passed_at: Time.now.utc }
 
       expect(assigns(:recovery_codes)).to eq otp_backup_codes
       expect(flash[:notice]).to eq 'Recovery codes successfully regenerated'
diff --git a/spec/controllers/settings/two_factor_authentications_controller_spec.rb b/spec/controllers/settings/two_factor_authentications_controller_spec.rb
index 922231ded..9df9763fd 100644
--- a/spec/controllers/settings/two_factor_authentications_controller_spec.rb
+++ b/spec/controllers/settings/two_factor_authentications_controller_spec.rb
@@ -58,7 +58,7 @@ describe Settings::TwoFactorAuthenticationsController do
       describe 'when creation succeeds' do
         it 'updates user secret' do
           before = user.otp_secret
-          post :create
+          post :create, session: { challenge_passed_at: Time.now.utc }
 
           expect(user.reload.otp_secret).not_to eq(before)
           expect(response).to redirect_to(new_settings_two_factor_authentication_confirmation_path)
diff --git a/spec/mailers/previews/user_mailer_preview.rb b/spec/mailers/previews/user_mailer_preview.rb
index ead3b3baa..464f177d0 100644
--- a/spec/mailers/previews/user_mailer_preview.rb
+++ b/spec/mailers/previews/user_mailer_preview.rb
@@ -18,6 +18,21 @@ class UserMailerPreview < ActionMailer::Preview
     UserMailer.password_change(User.first)
   end
 
+  # Preview this email at http://localhost:3000/rails/mailers/user_mailer/two_factor_disabled
+  def two_factor_disabled
+    UserMailer.two_factor_disabled(User.first)
+  end
+
+  # Preview this email at http://localhost:3000/rails/mailers/user_mailer/two_factor_enabled
+  def two_factor_enabled
+    UserMailer.two_factor_enabled(User.first)
+  end
+
+  # Preview this email at http://localhost:3000/rails/mailers/user_mailer/two_factor_recovery_codes_changed
+  def two_factor_recovery_codes_changed
+    UserMailer.two_factor_recovery_codes_changed(User.first)
+  end
+
   # Preview this email at http://localhost:3000/rails/mailers/user_mailer/reconfirmation_instructions
   def reconfirmation_instructions
     user = User.first
-- 
cgit 


From d930eb88b671fa6e5573fe7342bcdda87501bdb7 Mon Sep 17 00:00:00 2001
From: Eugen Rochko <eugen@zeonfederated.com>
Date: Thu, 19 Sep 2019 11:09:05 +0200
Subject: Add table of contents to about page (#11885)

Move public domain blocks information to about page
---
 app/controllers/about_controller.rb            |  43 +++-----
 app/javascript/styles/mastodon/about.scss      | 138 +++++++++++--------------
 app/javascript/styles/mastodon/containers.scss |  62 +++++++++++
 app/javascript/styles/mastodon/widgets.scss    |  83 ++++++++++-----
 app/lib/toc_generator.rb                       |  69 +++++++++++++
 app/models/domain_block.rb                     |   1 +
 app/views/about/blocks.html.haml               |  48 ---------
 app/views/about/more.html.haml                 |  59 ++++++++---
 config/locales/en.yml                          |  27 ++---
 config/routes.rb                               |   1 -
 10 files changed, 322 insertions(+), 209 deletions(-)
 create mode 100644 app/lib/toc_generator.rb
 delete mode 100644 app/views/about/blocks.html.haml

(limited to 'app/models')

diff --git a/app/controllers/about_controller.rb b/app/controllers/about_controller.rb
index 5e942e5c0..abd1ec0cb 100644
--- a/app/controllers/about_controller.rb
+++ b/app/controllers/about_controller.rb
@@ -3,9 +3,7 @@
 class AboutController < ApplicationController
   layout 'public'
 
-  before_action :require_open_federation!, only: [:show, :more, :blocks]
-  before_action :check_blocklist_enabled, only: [:blocks]
-  before_action :authenticate_user!, only: [:blocks], if: :blocklist_account_required?
+  before_action :require_open_federation!, only: [:show, :more]
   before_action :set_body_classes, only: :show
   before_action :set_instance_presenter
   before_action :set_expires_in, only: [:show, :more, :terms]
@@ -16,15 +14,20 @@ class AboutController < ApplicationController
 
   def more
     flash.now[:notice] = I18n.t('about.instance_actor_flash') if params[:instance_actor]
+
+    toc_generator = TOCGenerator.new(@instance_presenter.site_extended_description)
+
+    @contents          = toc_generator.html
+    @table_of_contents = toc_generator.toc
+    @blocks            = DomainBlock.with_user_facing_limitations.by_severity if display_blocks?
   end
 
   def terms; end
 
-  def blocks
-    @show_rationale = Setting.show_domain_blocks_rationale == 'all'
-    @show_rationale |= Setting.show_domain_blocks_rationale == 'users' && !current_user.nil? && current_user.functional?
-    @blocks = DomainBlock.with_user_facing_limitations.order('(CASE severity WHEN 0 THEN 1 WHEN 1 THEN 2 WHEN 2 THEN 0 END), reject_media, domain').to_a
-  end
+  helper_method :display_blocks?
+  helper_method :display_blocks_rationale?
+  helper_method :public_fetch_mode?
+  helper_method :new_user
 
   private
 
@@ -32,28 +35,14 @@ class AboutController < ApplicationController
     not_found if whitelist_mode?
   end
 
-  def check_blocklist_enabled
-    not_found if Setting.show_domain_blocks == 'disabled'
-  end
-
-  def blocklist_account_required?
-    Setting.show_domain_blocks == 'users'
+  def display_blocks?
+    Setting.show_domain_blocks == 'all' || (Setting.show_domain_blocks == 'users' && user_signed_in?)
   end
 
-  def block_severity_text(block)
-    if block.severity == 'suspend'
-      I18n.t('domain_blocks.suspension')
-    else
-      limitations = []
-      limitations << I18n.t('domain_blocks.media_block') if block.reject_media?
-      limitations << I18n.t('domain_blocks.silence') if block.severity == 'silence'
-      limitations.join(', ')
-    end
+  def display_blocks_rationale?
+    Setting.show_domain_blocks_rationale == 'all' || (Setting.show_domain_blocks_rationale == 'users' && user_signed_in?)
   end
 
-  helper_method :block_severity_text
-  helper_method :public_fetch_mode?
-
   def new_user
     User.new.tap do |user|
       user.build_account
@@ -61,8 +50,6 @@ class AboutController < ApplicationController
     end
   end
 
-  helper_method :new_user
-
   def set_instance_presenter
     @instance_presenter = InstancePresenter.new
   end
diff --git a/app/javascript/styles/mastodon/about.scss b/app/javascript/styles/mastodon/about.scss
index 61637ce96..c056ef85d 100644
--- a/app/javascript/styles/mastodon/about.scss
+++ b/app/javascript/styles/mastodon/about.scss
@@ -17,109 +17,102 @@ $small-breakpoint: 960px;
 
 .rich-formatting {
   font-family: $font-sans-serif, sans-serif;
-  font-size: 16px;
+  font-size: 14px;
   font-weight: 400;
-  font-size: 16px;
-  line-height: 30px;
+  line-height: 1.7;
+  word-wrap: break-word;
   color: $darker-text-color;
-  padding-right: 10px;
 
   a {
     color: $highlight-text-color;
     text-decoration: underline;
+
+    &:hover,
+    &:focus,
+    &:active {
+      text-decoration: none;
+    }
   }
 
   p,
   li {
-    font-family: $font-sans-serif, sans-serif;
-    font-size: 16px;
-    font-weight: 400;
-    font-size: 16px;
-    line-height: 30px;
-    margin-bottom: 12px;
     color: $darker-text-color;
+  }
 
-    a {
-      color: $highlight-text-color;
-      text-decoration: underline;
-    }
+  p {
+    margin-top: 0;
+    margin-bottom: .85em;
 
     &:last-child {
       margin-bottom: 0;
     }
   }
 
-  strong,
-  em {
+  strong {
     font-weight: 700;
-    color: lighten($darker-text-color, 10%);
+    color: $secondary-text-color;
   }
 
-  h1 {
-    font-family: $font-display, sans-serif;
-    font-size: 26px;
-    line-height: 30px;
-    font-weight: 500;
-    margin-bottom: 20px;
+  em {
+    font-style: italic;
     color: $secondary-text-color;
+  }
 
-    small {
-      font-family: $font-sans-serif, sans-serif;
-      display: block;
-      font-size: 18px;
-      font-weight: 400;
-      color: lighten($darker-text-color, 10%);
-    }
+  code {
+    font-size: 0.85em;
+    background: darken($ui-base-color, 8%);
+    border-radius: 4px;
+    padding: 0.2em 0.3em;
   }
 
-  h2 {
+  h1,
+  h2,
+  h3,
+  h4,
+  h5,
+  h6 {
     font-family: $font-display, sans-serif;
-    font-size: 22px;
-    line-height: 26px;
+    margin-top: 1.275em;
+    margin-bottom: .85em;
     font-weight: 500;
-    margin-bottom: 20px;
     color: $secondary-text-color;
   }
 
+  h1 {
+    font-size: 2em;
+  }
+
+  h2 {
+    font-size: 1.75em;
+  }
+
   h3 {
-    font-family: $font-display, sans-serif;
-    font-size: 18px;
-    line-height: 24px;
-    font-weight: 500;
-    margin-bottom: 20px;
-    color: $secondary-text-color;
+    font-size: 1.5em;
   }
 
   h4 {
-    font-family: $font-display, sans-serif;
-    font-size: 16px;
-    line-height: 24px;
-    font-weight: 500;
-    margin-bottom: 20px;
-    color: $secondary-text-color;
+    font-size: 1.25em;
   }
 
-  h5 {
-    font-family: $font-display, sans-serif;
-    font-size: 14px;
-    line-height: 24px;
-    font-weight: 500;
-    margin-bottom: 20px;
-    color: $secondary-text-color;
+  h5,
+  h6 {
+    font-size: 1em;
   }
 
-  h6 {
-    font-family: $font-display, sans-serif;
-    font-size: 12px;
-    line-height: 24px;
-    font-weight: 500;
-    margin-bottom: 20px;
-    color: $secondary-text-color;
+  ul {
+    list-style: disc;
+  }
+
+  ol {
+    list-style: decimal;
   }
 
   ul,
   ol {
-    margin-left: 20px;
+    margin: 0;
+    padding: 0;
+    padding-left: 2em;
+    margin-bottom: 0.85em;
 
     &[type='a'] {
       list-style-type: lower-alpha;
@@ -130,31 +123,22 @@ $small-breakpoint: 960px;
     }
   }
 
-  ul {
-    list-style: disc;
-  }
-
-  ol {
-    list-style: decimal;
-  }
-
-  li > ol,
-  li > ul {
-    margin-top: 6px;
-  }
-
   hr {
     width: 100%;
     height: 0;
     border: 0;
-    border-bottom: 1px solid rgba($ui-base-lighter-color, .6);
-    margin: 20px 0;
+    border-bottom: 1px solid lighten($ui-base-color, 4%);
+    margin: 1.7em 0;
 
     &.spacer {
       height: 1px;
       border: 0;
     }
   }
+
+  & > :first-child {
+    margin-top: 0;
+  }
 }
 
 .information-board {
@@ -416,7 +400,7 @@ $small-breakpoint: 960px;
   }
 
   &__call-to-action {
-    background: darken($ui-base-color, 4%);
+    background: $ui-base-color;
     border-radius: 4px;
     padding: 25px 40px;
     overflow: hidden;
diff --git a/app/javascript/styles/mastodon/containers.scss b/app/javascript/styles/mastodon/containers.scss
index aa45c0174..24bbf8211 100644
--- a/app/javascript/styles/mastodon/containers.scss
+++ b/app/javascript/styles/mastodon/containers.scss
@@ -141,6 +141,63 @@
     grid-row: 3;
   }
 
+  @media screen and (max-width: $no-gap-breakpoint) {
+    grid-gap: 0;
+    grid-template-columns: minmax(0, 100%);
+
+    .column-0 {
+      grid-column: 1;
+    }
+
+    .column-1 {
+      grid-column: 1;
+      grid-row: 3;
+    }
+
+    .column-2 {
+      grid-column: 1;
+      grid-row: 2;
+    }
+
+    .column-3 {
+      grid-column: 1;
+      grid-row: 4;
+    }
+  }
+}
+
+.grid-4 {
+  display: grid;
+  grid-gap: 10px;
+  grid-template-columns: 1fr 1fr 1fr 1fr;
+  grid-auto-columns: 25%;
+  grid-auto-rows: max-content;
+
+  .column-0 {
+    grid-column: 1 / 5;
+    grid-row: 1;
+  }
+
+  .column-1 {
+    grid-column: 1 / 4;
+    grid-row: 2;
+  }
+
+  .column-2 {
+    grid-column: 4;
+    grid-row: 2;
+  }
+
+  .column-3 {
+    grid-column: 2 / 5;
+    grid-row: 3;
+  }
+
+  .column-4 {
+    grid-column: 1;
+    grid-row: 3;
+  }
+
   .landing-page__call-to-action {
     min-height: 100%;
   }
@@ -189,6 +246,11 @@
     }
 
     .column-3 {
+      grid-column: 1;
+      grid-row: 5;
+    }
+
+    .column-4 {
       grid-column: 1;
       grid-row: 4;
     }
diff --git a/app/javascript/styles/mastodon/widgets.scss b/app/javascript/styles/mastodon/widgets.scss
index 04beb869c..ca050a8d9 100644
--- a/app/javascript/styles/mastodon/widgets.scss
+++ b/app/javascript/styles/mastodon/widgets.scss
@@ -128,41 +128,43 @@
   margin-bottom: 10px;
 }
 
-.contact-widget,
-.landing-page__information.contact-widget {
-  box-sizing: border-box;
-  padding: 20px;
-  min-height: 100%;
-  border-radius: 4px;
-  background: $ui-base-color;
-  box-shadow: 0 0 15px rgba($base-shadow-color, 0.2);
-}
-
 .contact-widget {
+  min-height: 100%;
   font-size: 15px;
   color: $darker-text-color;
   line-height: 20px;
   word-wrap: break-word;
   font-weight: 400;
+  padding: 0;
 
-  strong {
-    font-weight: 500;
+  h4 {
+    padding: 10px;
+    text-transform: uppercase;
+    font-weight: 700;
+    font-size: 13px;
+    color: $darker-text-color;
   }
 
-  p {
-    margin-bottom: 10px;
-
-    &:last-child {
-      margin-bottom: 0;
-    }
+  .account {
+    border-bottom: 0;
+    padding: 10px 0;
+    padding-top: 5px;
   }
 
-  &__mail {
-    margin-top: 10px;
+  & > a {
+    display: inline-block;
+    padding: 10px;
+    padding-top: 0;
+    color: $darker-text-color;
+    text-decoration: none;
+    white-space: nowrap;
+    overflow: hidden;
+    text-overflow: ellipsis;
 
-    a {
-      color: $primary-text-color;
-      text-decoration: none;
+    &:hover,
+    &:focus,
+    &:active {
+      text-decoration: underline;
     }
   }
 }
@@ -562,3 +564,38 @@ $fluid-breakpoint: $maximum-width + 20px;
     }
   }
 }
+
+.table-of-contents {
+  background: darken($ui-base-color, 4%);
+  min-height: 100%;
+  font-size: 14px;
+  border-radius: 4px;
+
+  li a {
+    display: block;
+    font-weight: 500;
+    padding: 15px;
+    overflow: hidden;
+    white-space: nowrap;
+    overflow: hidden;
+    text-overflow: ellipsis;
+    text-decoration: none;
+    color: $primary-text-color;
+    border-bottom: 1px solid lighten($ui-base-color, 4%);
+
+    &:hover,
+    &:focus,
+    &:active {
+      text-decoration: underline;
+    }
+  }
+
+  li:last-child a {
+    border-bottom: 0;
+  }
+
+  li ul {
+    padding-left: 20px;
+    border-bottom: 1px solid lighten($ui-base-color, 4%);
+  }
+}
diff --git a/app/lib/toc_generator.rb b/app/lib/toc_generator.rb
new file mode 100644
index 000000000..c6e179557
--- /dev/null
+++ b/app/lib/toc_generator.rb
@@ -0,0 +1,69 @@
+# frozen_string_literal: true
+
+class TOCGenerator
+  TARGET_ELEMENTS = %w(h1 h2 h3 h4 h5 h6).freeze
+  LISTED_ELEMENTS = %w(h2 h3).freeze
+
+  class Section
+    attr_accessor :depth, :title, :children, :anchor
+
+    def initialize(depth, title, anchor)
+      @depth    = depth
+      @title    = title
+      @children = []
+      @anchor   = anchor
+    end
+
+    delegate :<<, to: :children
+  end
+
+  def initialize(source_html)
+    @source_html = source_html
+    @processed   = false
+    @target_html = ''
+    @headers     = []
+    @slugs       = Hash.new { |h, k| h[k] = 0 }
+  end
+
+  def html
+    parse_and_transform unless @processed
+    @target_html
+  end
+
+  def toc
+    parse_and_transform unless @processed
+    @headers
+  end
+
+  private
+
+  def parse_and_transform
+    return if @source_html.blank?
+
+    parsed_html = Nokogiri::HTML.fragment(@source_html)
+
+    parsed_html.traverse do |node|
+      next unless TARGET_ELEMENTS.include?(node.name)
+
+      anchor = node.text.parameterize
+      @slugs[anchor] += 1
+      anchor = "#{anchor}-#{@slugs[anchor]}" if @slugs[anchor] > 1
+
+      node['id'] = anchor
+
+      next unless LISTED_ELEMENTS.include?(node.name)
+
+      depth          = node.name[1..-1]
+      latest_section = @headers.last
+
+      if latest_section.nil? || latest_section.depth >= depth
+        @headers << Section.new(depth, node.text, anchor)
+      else
+        latest_section << Section.new(depth, node.text, anchor)
+      end
+    end
+
+    @target_html = parsed_html.to_s
+    @processed   = true
+  end
+end
diff --git a/app/models/domain_block.rb b/app/models/domain_block.rb
index 4383cbd05..4e865b850 100644
--- a/app/models/domain_block.rb
+++ b/app/models/domain_block.rb
@@ -26,6 +26,7 @@ class DomainBlock < ApplicationRecord
 
   scope :matches_domain, ->(value) { where(arel_table[:domain].matches("%#{value}%")) }
   scope :with_user_facing_limitations, -> { where(severity: [:silence, :suspend]).or(where(reject_media: true)) }
+  scope :by_severity, -> { order(Arel.sql('(CASE severity WHEN 0 THEN 1 WHEN 1 THEN 2 WHEN 2 THEN 0 END), reject_media, domain')) }
 
   class << self
     def suspend?(domain)
diff --git a/app/views/about/blocks.html.haml b/app/views/about/blocks.html.haml
deleted file mode 100644
index a81a4d1eb..000000000
--- a/app/views/about/blocks.html.haml
+++ /dev/null
@@ -1,48 +0,0 @@
-- content_for :page_title do
-  = t('domain_blocks.title', instance: site_hostname)
-
-.grid
-  .column-0
-    .box-widget.rich-formatting
-      %h2= t('domain_blocks.blocked_domains')
-      %p= t('domain_blocks.description', instance: site_hostname)
-      .table-wrapper
-        %table.blocks-table
-          %thead
-            %tr
-              %th= t('domain_blocks.domain')
-              %th.severity-column= t('domain_blocks.severity')
-              - if @show_rationale
-                %th.button-column
-          %tbody
-            - if @blocks.empty?
-              %tr
-                %td{ colspan: @show_rationale ? 3 : 2 }= t('domain_blocks.no_domain_blocks')
-            - else
-              - @blocks.each_with_index do |block, i|
-                %tr{ class: i % 2 == 0 ? 'even': nil }
-                  %td{ title: block.domain }= block.domain
-                  %td= block_severity_text(block)
-                  - if @show_rationale
-                    %td
-                      - if block.public_comment.present?
-                        %button.icon-button{ title: t('domain_blocks.show_rationale'), 'aria-label' => t('domain_blocks.show_rationale') }
-                          = fa_icon 'chevron-down fw', 'aria-hidden' => true
-                - if @show_rationale
-                  - if block.public_comment.present?
-                    %tr.rationale.hidden
-                      %td{ colspan: 3 }= block.public_comment.presence
-      %h2= t('domain_blocks.severity_legend.title')
-      - if @blocks.any? { |block| block.reject_media? }
-        %h3= t('domain_blocks.media_block')
-        %p= t('domain_blocks.severity_legend.media_block')
-      - if @blocks.any? { |block| block.severity == 'silence' }
-        %h3= t('domain_blocks.silence')
-        %p= t('domain_blocks.severity_legend.silence')
-      - if @blocks.any? { |block| block.severity == 'suspend' }
-        %h3= t('domain_blocks.suspension')
-        %p= t('domain_blocks.severity_legend.suspension')
-        - if public_fetch_mode?
-          %p= t('domain_blocks.severity_legend.suspension_disclaimer')
-  .column-1
-    = render 'application/sidebar'
diff --git a/app/views/about/more.html.haml b/app/views/about/more.html.haml
index 21431ef8e..4b3035ee8 100644
--- a/app/views/about/more.html.haml
+++ b/app/views/about/more.html.haml
@@ -5,7 +5,7 @@
   = javascript_pack_tag 'public', integrity: true, crossorigin: 'anonymous'
   = render partial: 'shared/og'
 
-.grid-3
+.grid-4
   .column-0
     .public-account-header.public-account-header--no-bar
       .public-account-header__image
@@ -28,22 +28,57 @@
             = image_tag @instance_presenter.mascot&.file&.url || asset_pack_path('media/images/elephant_ui_plane.svg'), alt: ''
 
   .column-2
-    .landing-page__information.contact-widget
-      %p
-        %strong= t 'about.administered_by'
+    .contact-widget
+      %h4= t 'about.administered_by'
 
       = account_link_to(@instance_presenter.contact_account)
 
       - if @instance_presenter.site_contact_email.present?
-        %p.contact-widget__mail
-          %strong
-            = succeed ':' do
-              = t 'about.contact'
-          %br/
-          = mail_to @instance_presenter.site_contact_email, nil, title: @instance_presenter.site_contact_email
+        %h4
+          = succeed ':' do
+            = t 'about.contact'
+
+        = mail_to @instance_presenter.site_contact_email, nil, title: @instance_presenter.site_contact_email
 
   .column-3
     = render 'application/flashes'
 
-    .box-widget
-      .rich-formatting= @instance_presenter.site_extended_description.html_safe.presence || t('about.extended_description_html')
+    - if @contents.blank? && (!display_blocks? || @blocks&.empty?)
+      = nothing_here
+    - else
+      .box-widget
+        .rich-formatting
+          = @contents.html_safe
+
+          - if display_blocks? && !@blocks.empty?
+            %h2#unavailable-content= t('about.unavailable_content')
+
+            %p= t('about.unavailable_content_html')
+
+            - @blocks.each do |domain_block|
+              %p
+                %strong= "#{domain_block.domain}:"
+
+                - if domain_block.suspend?
+                  = t('about.unavailable_content_description.suspended')
+                - else
+                  = t('about.unavailable_content_description.silenced') if domain_block.silence?
+                  = t('about.unavailable_content_description.rejecting_media') if domain_block.reject_media?
+
+                - if display_blocks_rationale?
+                  %strong= t('about.unavailable_content_description.reason')
+                  = domain_block.public_comment
+
+  .column-4
+    %ul.table-of-contents
+      - @table_of_contents.each do |item|
+        %li
+          = link_to item.title, "##{item.anchor}"
+
+          - unless item.children.empty?
+            %ul
+              - item.children.each do |sub_item|
+                %li= link_to sub_item.title, "##{sub_item.anchor}"
+
+      - if display_blocks? && !@blocks.empty?
+        %li= link_to t('about.unavailable_content'), '#unavailable-content'
diff --git a/config/locales/en.yml b/config/locales/en.yml
index da06b0e51..dabb679e7 100644
--- a/config/locales/en.yml
+++ b/config/locales/en.yml
@@ -17,9 +17,6 @@ en:
     contact_unavailable: N/A
     discover_users: Discover users
     documentation: Documentation
-    extended_description_html: |
-      <h3>A good place for rules</h3>
-      <p>The extended description has not been set up yet.</p>
     federation_hint_html: With an account on %{instance} you'll be able to follow people on any Mastodon server and beyond.
     generic_description: "%{domain} is one server in the network"
     get_apps: Try a mobile app
@@ -38,6 +35,13 @@ en:
     status_count_before: Who authored
     tagline: Follow friends and discover new ones
     terms: Terms of service
+    unavailable_content: Unavailable content
+    unavailable_content_description:
+      reason: 'Reason:'
+      rejecting_media: Media files from this server will not be processed and and no thumbnails will be displayed, requiring manual click-through to the other server.
+      silenced: Posts from this server will not show up anywhere except your home feed if you follow the author.
+      suspended: You won't be able to follow anyone from this server, and no data from it will be processed or stored, and no data exchanged.
+    unavailable_content_html: Mastodon generally allows you to view content from and interact with users from any other server in the fediverse. These are the exceptions that have been made on this particular server.
     user_count_after:
       one: user
       other: users
@@ -661,23 +665,6 @@ en:
     directory: Profile directory
     explanation: Discover users based on their interests
     explore_mastodon: Explore %{title}
-  domain_blocks:
-    blocked_domains: List of limited and blocked domains
-    description: This is the list of servers that %{instance} limits or reject federation with.
-    domain: Domain
-    media_block: Media block
-    no_domain_blocks: "(No domain blocks)"
-    severity: Severity
-    severity_legend:
-      media_block: Media files coming from the server are neither fetched, stored, or displayed to the user.
-      silence: Accounts from silenced servers can be found, followed and interacted with, but their toots will not appear in the public timelines, and notifications from them will not reach local users who are not following them.
-      suspension: No content from suspended servers is stored or displayed, nor is any content sent to them. Interactions from suspended servers are ignored.
-      suspension_disclaimer: Suspended servers may occasionally retrieve public content from this server.
-      title: Severities
-    show_rationale: Show rationale
-    silence: Silence
-    suspension: Suspension
-    title: "%{instance} List of blocked instances"
   domain_validator:
     invalid_domain: is not a valid domain name
   errors:
diff --git a/config/routes.rb b/config/routes.rb
index 9ad1ea65d..dcfa079a0 100644
--- a/config/routes.rb
+++ b/config/routes.rb
@@ -441,7 +441,6 @@ Rails.application.routes.draw do
 
   get '/about',        to: 'about#show'
   get '/about/more',   to: 'about#more'
-  get '/about/blocks', to: 'about#blocks'
   get '/terms',        to: 'about#terms'
 
   match '/', via: [:post, :put, :patch, :delete], to: 'application#raise_not_found', format: false
-- 
cgit 


From 3ed94dcc1acf73f1d0d1ab43567b88ee953f57c9 Mon Sep 17 00:00:00 2001
From: Eugen Rochko <eugen@zeonfederated.com>
Date: Thu, 19 Sep 2019 20:58:19 +0200
Subject: Add account migration UI (#11846)

Fix #10736

- Change data export to be available for non-functional accounts
- Change non-functional accounts to include redirecting accounts
---
 .../concerns/export_controller_concern.rb          |  7 ++
 app/controllers/settings/aliases_controller.rb     | 42 +++++++++++
 app/controllers/settings/exports_controller.rb     |  7 ++
 app/controllers/settings/migrations_controller.rb  | 48 ++++++++++---
 app/helpers/settings_helper.rb                     |  8 +++
 app/models/account_alias.rb                        | 41 +++++++++++
 app/models/account_migration.rb                    | 74 +++++++++++++++++++
 app/models/concerns/account_associations.rb        |  2 +
 app/models/form/migration.rb                       | 25 -------
 app/models/remote_follow.rb                        |  2 +-
 app/models/user.rb                                 |  2 +-
 app/serializers/activitypub/move_serializer.rb     | 26 +++++++
 app/views/auth/registrations/_status.html.haml     | 30 ++++----
 app/views/auth/registrations/edit.html.haml        |  2 +-
 app/views/settings/aliases/index.html.haml         | 29 ++++++++
 app/views/settings/exports/show.html.haml          |  4 ++
 app/views/settings/migrations/show.html.haml       | 84 +++++++++++++++++++---
 app/views/settings/profiles/show.html.haml         |  5 ++
 .../activitypub/move_distribution_worker.rb        | 32 +++++++++
 config/locales/en.yml                              | 38 ++++++++--
 config/locales/simple_form.en.yml                  | 10 +++
 config/navigation.rb                               |  8 +--
 config/routes.rb                                   |  8 ++-
 .../20190914202517_create_account_migrations.rb    | 12 ++++
 .../20190915194355_create_account_aliases.rb       | 11 +++
 db/schema.rb                                       | 23 ++++++
 .../settings/migrations_controller_spec.rb         | 14 ++--
 spec/fabricators/account_alias_fabricator.rb       |  5 ++
 spec/fabricators/account_migration_fabricator.rb   |  6 ++
 spec/models/account_alias_spec.rb                  |  5 ++
 spec/models/account_migration_spec.rb              |  5 ++
 31 files changed, 542 insertions(+), 73 deletions(-)
 create mode 100644 app/controllers/settings/aliases_controller.rb
 create mode 100644 app/models/account_alias.rb
 create mode 100644 app/models/account_migration.rb
 delete mode 100644 app/models/form/migration.rb
 create mode 100644 app/serializers/activitypub/move_serializer.rb
 create mode 100644 app/views/settings/aliases/index.html.haml
 create mode 100644 app/workers/activitypub/move_distribution_worker.rb
 create mode 100644 db/migrate/20190914202517_create_account_migrations.rb
 create mode 100644 db/migrate/20190915194355_create_account_aliases.rb
 create mode 100644 spec/fabricators/account_alias_fabricator.rb
 create mode 100644 spec/fabricators/account_migration_fabricator.rb
 create mode 100644 spec/models/account_alias_spec.rb
 create mode 100644 spec/models/account_migration_spec.rb

(limited to 'app/models')

diff --git a/app/controllers/concerns/export_controller_concern.rb b/app/controllers/concerns/export_controller_concern.rb
index e20b71a30..bfe990c82 100644
--- a/app/controllers/concerns/export_controller_concern.rb
+++ b/app/controllers/concerns/export_controller_concern.rb
@@ -5,7 +5,10 @@ module ExportControllerConcern
 
   included do
     before_action :authenticate_user!
+    before_action :require_not_suspended!
     before_action :load_export
+
+    skip_before_action :require_functional!
   end
 
   private
@@ -27,4 +30,8 @@ module ExportControllerConcern
   def export_filename
     "#{controller_name}.csv"
   end
+
+  def require_not_suspended!
+    forbidden if current_account.suspended?
+  end
 end
diff --git a/app/controllers/settings/aliases_controller.rb b/app/controllers/settings/aliases_controller.rb
new file mode 100644
index 000000000..2b675f065
--- /dev/null
+++ b/app/controllers/settings/aliases_controller.rb
@@ -0,0 +1,42 @@
+# frozen_string_literal: true
+
+class Settings::AliasesController < Settings::BaseController
+  layout 'admin'
+
+  before_action :authenticate_user!
+  before_action :set_aliases, except: :destroy
+  before_action :set_alias, only: :destroy
+
+  def index
+    @alias = current_account.aliases.build
+  end
+
+  def create
+    @alias = current_account.aliases.build(resource_params)
+
+    if @alias.save
+      redirect_to settings_aliases_path, notice: I18n.t('aliases.created_msg')
+    else
+      render :show
+    end
+  end
+
+  def destroy
+    @alias.destroy!
+    redirect_to settings_aliases_path, notice: I18n.t('aliases.deleted_msg')
+  end
+
+  private
+
+  def resource_params
+    params.require(:account_alias).permit(:acct)
+  end
+
+  def set_alias
+    @alias = current_account.aliases.find(params[:id])
+  end
+
+  def set_aliases
+    @aliases = current_account.aliases.order(id: :desc).reject(&:new_record?)
+  end
+end
diff --git a/app/controllers/settings/exports_controller.rb b/app/controllers/settings/exports_controller.rb
index 3012fbf77..0e93d07a9 100644
--- a/app/controllers/settings/exports_controller.rb
+++ b/app/controllers/settings/exports_controller.rb
@@ -6,6 +6,9 @@ class Settings::ExportsController < Settings::BaseController
   layout 'admin'
 
   before_action :authenticate_user!
+  before_action :require_not_suspended!
+
+  skip_before_action :require_functional!
 
   def show
     @export  = Export.new(current_account)
@@ -34,4 +37,8 @@ class Settings::ExportsController < Settings::BaseController
   def lock_options
     { redis: Redis.current, key: "backup:#{current_user.id}" }
   end
+
+  def require_not_suspended!
+    forbidden if current_account.suspended?
+  end
 end
diff --git a/app/controllers/settings/migrations_controller.rb b/app/controllers/settings/migrations_controller.rb
index 59eb48779..90092c692 100644
--- a/app/controllers/settings/migrations_controller.rb
+++ b/app/controllers/settings/migrations_controller.rb
@@ -4,31 +4,59 @@ class Settings::MigrationsController < Settings::BaseController
   layout 'admin'
 
   before_action :authenticate_user!
+  before_action :require_not_suspended!
+  before_action :set_migrations
+  before_action :set_cooldown
+
+  skip_before_action :require_functional!
 
   def show
-    @migration = Form::Migration.new(account: current_account.moved_to_account)
+    @migration = current_account.migrations.build
   end
 
-  def update
-    @migration = Form::Migration.new(resource_params)
+  def create
+    @migration = current_account.migrations.build(resource_params)
 
-    if @migration.valid? && migration_account_changed?
-      current_account.update!(moved_to_account: @migration.account)
+    if @migration.save_with_challenge(current_user)
+      current_account.update!(moved_to_account: @migration.target_account)
       ActivityPub::UpdateDistributionWorker.perform_async(current_account.id)
-      redirect_to settings_migration_path, notice: I18n.t('migrations.updated_msg')
+      ActivityPub::MoveDistributionWorker.perform_async(@migration.id)
+      redirect_to settings_migration_path, notice: I18n.t('migrations.moved_msg', acct: current_account.moved_to_account.acct)
     else
       render :show
     end
   end
 
+  def cancel
+    if current_account.moved_to_account_id.present?
+      current_account.update!(moved_to_account: nil)
+      ActivityPub::UpdateDistributionWorker.perform_async(current_account.id)
+    end
+
+    redirect_to settings_migration_path, notice: I18n.t('migrations.cancelled_msg')
+  end
+
+  helper_method :on_cooldown?
+
   private
 
   def resource_params
-    params.require(:migration).permit(:acct)
+    params.require(:account_migration).permit(:acct, :current_password, :current_username)
+  end
+
+  def set_migrations
+    @migrations = current_account.migrations.includes(:target_account).order(id: :desc).reject(&:new_record?)
+  end
+
+  def set_cooldown
+    @cooldown = current_account.migrations.within_cooldown.first
+  end
+
+  def on_cooldown?
+    @cooldown.present?
   end
 
-  def migration_account_changed?
-    current_account.moved_to_account_id != @migration.account&.id &&
-      current_account.id != @migration.account&.id
+  def require_not_suspended!
+    forbidden if current_account.suspended?
   end
 end
diff --git a/app/helpers/settings_helper.rb b/app/helpers/settings_helper.rb
index 2b3fd1263..ecc73baf5 100644
--- a/app/helpers/settings_helper.rb
+++ b/app/helpers/settings_helper.rb
@@ -87,4 +87,12 @@ module SettingsHelper
       'desktop'
     end
   end
+
+  def compact_account_link_to(account)
+    return if account.nil?
+
+    link_to ActivityPub::TagManager.instance.url_for(account), class: 'name-tag', title: account.acct do
+      safe_join([image_tag(account.avatar.url, width: 15, height: 15, alt: display_name(account), class: 'avatar'), content_tag(:span, account.acct, class: 'username')], ' ')
+    end
+  end
 end
diff --git a/app/models/account_alias.rb b/app/models/account_alias.rb
new file mode 100644
index 000000000..e9a0dd79e
--- /dev/null
+++ b/app/models/account_alias.rb
@@ -0,0 +1,41 @@
+# frozen_string_literal: true
+
+# == Schema Information
+#
+# Table name: account_aliases
+#
+#  id         :bigint(8)        not null, primary key
+#  account_id :bigint(8)
+#  acct       :string           default(""), not null
+#  uri        :string           default(""), not null
+#  created_at :datetime         not null
+#  updated_at :datetime         not null
+#
+
+class AccountAlias < ApplicationRecord
+  belongs_to :account
+
+  validates :acct, presence: true, domain: { acct: true }
+  validates :uri, presence: true
+
+  before_validation :set_uri
+  after_create :add_to_account
+  after_destroy :remove_from_account
+
+  private
+
+  def set_uri
+    target_account = ResolveAccountService.new.call(acct)
+    self.uri       = ActivityPub::TagManager.instance.uri_for(target_account) unless target_account.nil?
+  rescue Goldfinger::Error, HTTP::Error, OpenSSL::SSL::SSLError, Mastodon::Error
+    # Validation will take care of it
+  end
+
+  def add_to_account
+    account.update(also_known_as: account.also_known_as + [uri])
+  end
+
+  def remove_from_account
+    account.update(also_known_as: account.also_known_as.reject { |x| x == uri })
+  end
+end
diff --git a/app/models/account_migration.rb b/app/models/account_migration.rb
new file mode 100644
index 000000000..15830bffb
--- /dev/null
+++ b/app/models/account_migration.rb
@@ -0,0 +1,74 @@
+# frozen_string_literal: true
+
+# == Schema Information
+#
+# Table name: account_migrations
+#
+#  id                :bigint(8)        not null, primary key
+#  account_id        :bigint(8)
+#  acct              :string           default(""), not null
+#  followers_count   :bigint(8)        default(0), not null
+#  target_account_id :bigint(8)
+#  created_at        :datetime         not null
+#  updated_at        :datetime         not null
+#
+
+class AccountMigration < ApplicationRecord
+  COOLDOWN_PERIOD = 30.days.freeze
+
+  belongs_to :account
+  belongs_to :target_account, class_name: 'Account'
+
+  before_validation :set_target_account
+  before_validation :set_followers_count
+
+  validates :acct, presence: true, domain: { acct: true }
+  validate :validate_migration_cooldown
+  validate :validate_target_account
+
+  scope :within_cooldown, ->(now = Time.now.utc) { where(arel_table[:created_at].gteq(now - COOLDOWN_PERIOD)) }
+
+  attr_accessor :current_password, :current_username
+
+  def save_with_challenge(current_user)
+    if current_user.encrypted_password.present?
+      errors.add(:current_password, :invalid) unless current_user.valid_password?(current_password)
+    else
+      errors.add(:current_username, :invalid) unless account.username == current_username
+    end
+
+    return false unless errors.empty?
+
+    save
+  end
+
+  def cooldown_at
+    created_at + COOLDOWN_PERIOD
+  end
+
+  private
+
+  def set_target_account
+    self.target_account = ResolveAccountService.new.call(acct)
+  rescue Goldfinger::Error, HTTP::Error, OpenSSL::SSL::SSLError, Mastodon::Error
+    # Validation will take care of it
+  end
+
+  def set_followers_count
+    self.followers_count = account.followers_count
+  end
+
+  def validate_target_account
+    if target_account.nil?
+      errors.add(:acct, I18n.t('migrations.errors.not_found'))
+    else
+      errors.add(:acct, I18n.t('migrations.errors.missing_also_known_as')) unless target_account.also_known_as.include?(ActivityPub::TagManager.instance.uri_for(account))
+      errors.add(:acct, I18n.t('migrations.errors.already_moved')) if account.moved_to_account_id.present? && account.moved_to_account_id == target_account.id
+      errors.add(:acct, I18n.t('migrations.errors.move_to_self')) if account.id == target_account.id
+    end
+  end
+
+  def validate_migration_cooldown
+    errors.add(:base, I18n.t('migrations.errors.on_cooldown')) if account.migrations.within_cooldown.exists?
+  end
+end
diff --git a/app/models/concerns/account_associations.rb b/app/models/concerns/account_associations.rb
index 1db7771c7..c9cc5c610 100644
--- a/app/models/concerns/account_associations.rb
+++ b/app/models/concerns/account_associations.rb
@@ -52,6 +52,8 @@ module AccountAssociations
 
     # Account migrations
     belongs_to :moved_to_account, class_name: 'Account', optional: true
+    has_many :migrations, class_name: 'AccountMigration', dependent: :destroy, inverse_of: :account
+    has_many :aliases, class_name: 'AccountAlias', dependent: :destroy, inverse_of: :account
 
     # Hashtags
     has_and_belongs_to_many :tags
diff --git a/app/models/form/migration.rb b/app/models/form/migration.rb
deleted file mode 100644
index c2a8655e1..000000000
--- a/app/models/form/migration.rb
+++ /dev/null
@@ -1,25 +0,0 @@
-# frozen_string_literal: true
-
-class Form::Migration
-  include ActiveModel::Validations
-
-  attr_accessor :acct, :account
-
-  def initialize(attrs = {})
-    @account = attrs[:account]
-    @acct    = attrs[:account].acct unless @account.nil?
-    @acct    = attrs[:acct].gsub(/\A@/, '').strip unless attrs[:acct].nil?
-  end
-
-  def valid?
-    return false unless super
-    set_account
-    errors.empty?
-  end
-
-  private
-
-  def set_account
-    self.account = (ResolveAccountService.new.call(acct) if account.nil? && acct.present?)
-  end
-end
diff --git a/app/models/remote_follow.rb b/app/models/remote_follow.rb
index 52dd3f67b..5ea535287 100644
--- a/app/models/remote_follow.rb
+++ b/app/models/remote_follow.rb
@@ -49,7 +49,7 @@ class RemoteFollow
   end
 
   def fetch_template!
-    return missing_resource if acct.blank?
+    return missing_resource_error if acct.blank?
 
     _, domain = acct.split('@')
 
diff --git a/app/models/user.rb b/app/models/user.rb
index b48455802..9a19a53b3 100644
--- a/app/models/user.rb
+++ b/app/models/user.rb
@@ -168,7 +168,7 @@ class User < ApplicationRecord
   end
 
   def functional?
-    confirmed? && approved? && !disabled? && !account.suspended?
+    confirmed? && approved? && !disabled? && !account.suspended? && account.moved_to_account_id.nil?
   end
 
   def unconfirmed_or_pending?
diff --git a/app/serializers/activitypub/move_serializer.rb b/app/serializers/activitypub/move_serializer.rb
new file mode 100644
index 000000000..5675875fa
--- /dev/null
+++ b/app/serializers/activitypub/move_serializer.rb
@@ -0,0 +1,26 @@
+# frozen_string_literal: true
+
+class ActivityPub::MoveSerializer < ActivityPub::Serializer
+  attributes :id, :type, :target, :actor
+  attribute :virtual_object, key: :object
+
+  def id
+    [ActivityPub::TagManager.instance.uri_for(object.account), '#moves/', object.id].join
+  end
+
+  def type
+    'Move'
+  end
+
+  def target
+    ActivityPub::TagManager.instance.uri_for(object.target_account)
+  end
+
+  def virtual_object
+    ActivityPub::TagManager.instance.uri_for(object.account)
+  end
+
+  def actor
+    ActivityPub::TagManager.instance.uri_for(object.account)
+  end
+end
diff --git a/app/views/auth/registrations/_status.html.haml b/app/views/auth/registrations/_status.html.haml
index b38a83d67..47112dae0 100644
--- a/app/views/auth/registrations/_status.html.haml
+++ b/app/views/auth/registrations/_status.html.haml
@@ -1,16 +1,22 @@
 %h3= t('auth.status.account_status')
 
-- if @user.account.suspended?
-  %span.negative-hint= t('user_mailer.warning.explanation.suspend')
-- elsif @user.disabled?
-  %span.negative-hint= t('user_mailer.warning.explanation.disable')
-- elsif @user.account.silenced?
-  %span.warning-hint= t('user_mailer.warning.explanation.silence')
-- elsif !@user.confirmed?
-  %span.warning-hint= t('auth.status.confirming')
-- elsif !@user.approved?
-  %span.warning-hint= t('auth.status.pending')
-- else
-  %span.positive-hint= t('auth.status.functional')
+.simple_form
+  %p.hint
+    - if @user.account.suspended?
+      %span.negative-hint= t('user_mailer.warning.explanation.suspend')
+    - elsif @user.disabled?
+      %span.negative-hint= t('user_mailer.warning.explanation.disable')
+    - elsif @user.account.silenced?
+      %span.warning-hint= t('user_mailer.warning.explanation.silence')
+    - elsif !@user.confirmed?
+      %span.warning-hint= t('auth.status.confirming')
+      = link_to t('auth.didnt_get_confirmation'), new_user_confirmation_path
+    - elsif !@user.approved?
+      %span.warning-hint= t('auth.status.pending')
+    - elsif @user.account.moved_to_account_id.present?
+      %span.positive-hint= t('auth.status.redirecting_to', acct: @user.account.moved_to_account.acct)
+      = link_to t('migrations.cancel'), settings_migration_path
+    - else
+      %span.positive-hint= t('auth.status.functional')
 
 %hr.spacer/
diff --git a/app/views/auth/registrations/edit.html.haml b/app/views/auth/registrations/edit.html.haml
index 710ee5c68..885171c58 100644
--- a/app/views/auth/registrations/edit.html.haml
+++ b/app/views/auth/registrations/edit.html.haml
@@ -13,7 +13,7 @@
       .fields-row__column.fields-group.fields-row__column-6
         = f.input :email, wrapper: :with_label, input_html: { 'aria-label' => t('simple_form.labels.defaults.email') }, required: true, disabled: current_account.suspended?
       .fields-row__column.fields-group.fields-row__column-6
-        = f.input :current_password, wrapper: :with_label, input_html: { 'aria-label' => t('simple_form.labels.defaults.current_password'), :autocomplete => 'off' }, required: true, disabled: current_account.suspended?
+        = f.input :current_password, wrapper: :with_label, input_html: { 'aria-label' => t('simple_form.labels.defaults.current_password'), :autocomplete => 'off' }, required: true, disabled: current_account.suspended?, hint: false
 
     .fields-row
       .fields-row__column.fields-group.fields-row__column-6
diff --git a/app/views/settings/aliases/index.html.haml b/app/views/settings/aliases/index.html.haml
new file mode 100644
index 000000000..5b6986368
--- /dev/null
+++ b/app/views/settings/aliases/index.html.haml
@@ -0,0 +1,29 @@
+- content_for :page_title do
+  = t('settings.aliases')
+
+= simple_form_for @alias, url: settings_aliases_path do |f|
+  = render 'shared/error_messages', object: @alias
+
+  %p.hint= t('aliases.hint_html')
+
+  %hr.spacer/
+
+  .fields-group
+    = f.input :acct, wrapper: :with_block_label, input_html: { autocapitalize: 'none', autocorrect: 'off' }
+
+  .actions
+    = f.button :button, t('aliases.add_new'), type: :submit, class: 'button'
+
+%hr.spacer/
+
+.table-wrapper
+  %table.table.inline-table
+    %thead
+      %tr
+        %th= t('simple_form.labels.account_alias.acct')
+        %th
+    %tbody
+      - @aliases.each do |account_alias|
+        %tr
+          %td= account_alias.acct
+          %td= table_link_to 'trash', t('aliases.remove'), settings_alias_path(account_alias), data: { method: :delete }
diff --git a/app/views/settings/exports/show.html.haml b/app/views/settings/exports/show.html.haml
index b13cea976..76ff76bd9 100644
--- a/app/views/settings/exports/show.html.haml
+++ b/app/views/settings/exports/show.html.haml
@@ -37,12 +37,16 @@
         %td= number_with_delimiter @export.total_domain_blocks
         %td= table_link_to 'download', t('exports.csv'), settings_exports_domain_blocks_path(format: :csv)
 
+%hr.spacer/
+
 %p.muted-hint= t('exports.archive_takeout.hint_html')
 
 - if policy(:backup).create?
   %p= link_to t('exports.archive_takeout.request'), settings_export_path, class: 'button', method: :post
 
 - unless @backups.empty?
+  %hr.spacer/
+
   .table-wrapper
     %table.table
       %thead
diff --git a/app/views/settings/migrations/show.html.haml b/app/views/settings/migrations/show.html.haml
index c69061d50..1e5c47726 100644
--- a/app/views/settings/migrations/show.html.haml
+++ b/app/views/settings/migrations/show.html.haml
@@ -1,17 +1,85 @@
 - content_for :page_title do
   = t('settings.migrate')
 
-= simple_form_for @migration, as: :migration, url: settings_migration_path, html: { method: :put } do |f|
-  - if @migration.account
-    %p.hint= t('migrations.currently_redirecting')
+.simple_form
+  - if current_account.moved_to_account.present?
+    .fields-row
+      .fields-row__column.fields-group.fields-row__column-6
+        = render 'application/card', account: current_account.moved_to_account
+      .fields-row__column.fields-group.fields-row__column-6
+        %p.hint
+          %span.positive-hint= t('migrations.redirecting_to', acct: current_account.moved_to_account.acct)
 
-    .fields-group
-      = render partial: 'application/card', locals: { account: @migration.account }
+        %p.hint= t('migrations.cancel_explanation')
+
+        %p.hint= link_to t('migrations.cancel'), cancel_settings_migration_path, data: { method: :post }
+  - else
+    %p.hint
+      %span.positive-hint= t('migrations.not_redirecting')
+
+%hr.spacer/
+
+%h3= t 'migrations.proceed_with_move'
+
+= simple_form_for @migration, url: settings_migration_path do |f|
+  - if on_cooldown?
+    %span.warning-hint= t('migrations.on_cooldown', count: ((@cooldown.cooldown_at - Time.now.utc) / 1.day.seconds).ceil)
+  - else
+    %p.hint= t('migrations.warning.before')
+
+    %ul.hint
+      %li.warning-hint= t('migrations.warning.followers')
+      %li.warning-hint= t('migrations.warning.other_data')
+      %li.warning-hint= t('migrations.warning.backreference_required')
+      %li.warning-hint= t('migrations.warning.cooldown')
+      %li.warning-hint= t('migrations.warning.disabled_account')
+
+  %hr.spacer/
 
   = render 'shared/error_messages', object: @migration
 
-  .fields-group
-    = f.input :acct, placeholder: t('migrations.acct')
+  .fields-row
+    .fields-row__column.fields-group.fields-row__column-6
+      = f.input :acct, wrapper: :with_block_label, input_html: { autocapitalize: 'none', autocorrect: 'off' }, disabled: on_cooldown?
+
+    .fields-row__column.fields-group.fields-row__column-6
+      - if current_user.encrypted_password.present?
+        = f.input :current_password, wrapper: :with_block_label, input_html: { :autocomplete => 'off' }, required: true, disabled: on_cooldown?
+      - else
+        = f.input :current_username, wrapper: :with_block_label, input_html: { :autocomplete => 'off' }, required: true, disabled: on_cooldown?
 
   .actions
-    = f.button :button, t('migrations.proceed'), type: :submit, class: 'negative'
+    = f.button :button, t('migrations.proceed_with_move'), type: :submit, class: 'button button--destructive', disabled: on_cooldown?
+
+- unless @migrations.empty?
+  %hr.spacer/
+
+  %h3= t 'migrations.past_migrations'
+
+  %hr.spacer/
+
+  .table-wrapper
+    %table.table.inline-table
+      %thead
+        %tr
+          %th= t('migrations.acct')
+          %th= t('migrations.followers_count')
+          %th
+      %tbody
+        - @migrations.each do |migration|
+          %tr
+            %td
+              - if migration.target_account.present?
+                = compact_account_link_to migration.target_account
+              - else
+                = migration.acct
+
+            %td= number_with_delimiter migration.followers_count
+
+            %td
+              %time.time-ago{ datetime: migration.created_at.iso8601, title: l(migration.created_at) }= l(migration.created_at)
+
+%hr.spacer/
+
+%h3= t 'migrations.incoming_migrations'
+%p.muted-hint= t('migrations.incoming_migrations_html', path: settings_aliases_path)
diff --git a/app/views/settings/profiles/show.html.haml b/app/views/settings/profiles/show.html.haml
index f042011d6..6929f54f3 100644
--- a/app/views/settings/profiles/show.html.haml
+++ b/app/views/settings/profiles/show.html.haml
@@ -60,6 +60,11 @@
 %h6= t('auth.migrate_account')
 %p.muted-hint= t('auth.migrate_account_html', path: settings_migration_path)
 
+%hr.spacer/
+
+%h6= t 'migrations.incoming_migrations'
+%p.muted-hint= t('migrations.incoming_migrations_html', path: settings_aliases_path)
+
 - if open_deletion?
   %hr.spacer/
 
diff --git a/app/workers/activitypub/move_distribution_worker.rb b/app/workers/activitypub/move_distribution_worker.rb
new file mode 100644
index 000000000..396d5258f
--- /dev/null
+++ b/app/workers/activitypub/move_distribution_worker.rb
@@ -0,0 +1,32 @@
+# frozen_string_literal: true
+
+class ActivityPub::MoveDistributionWorker
+  include Sidekiq::Worker
+  include Payloadable
+
+  sidekiq_options queue: 'push'
+
+  def perform(migration_id)
+    @migration = AccountMigration.find(migration_id)
+
+    ActivityPub::DeliveryWorker.push_bulk(inboxes) do |inbox_url|
+      [signed_payload, @account.id, inbox_url]
+    end
+
+    ActivityPub::DeliveryWorker.push_bulk(Relay.enabled.pluck(:inbox_url)) do |inbox_url|
+      [signed_payload, @account.id, inbox_url]
+    end
+  rescue ActiveRecord::RecordNotFound
+    true
+  end
+
+  private
+
+  def inboxes
+    @inboxes ||= @migration.account.followers.inboxes
+  end
+
+  def signed_payload
+    @signed_payload ||= Oj.dump(serialize_payload(@migration, ActivityPub::MoveSerializer, signer: @account))
+  end
+end
diff --git a/config/locales/en.yml b/config/locales/en.yml
index dabb679e7..c29c7f871 100644
--- a/config/locales/en.yml
+++ b/config/locales/en.yml
@@ -554,6 +554,12 @@ en:
     new_trending_tag:
       body: 'The hashtag #%{name} is trending today, but has not been previously reviewed. It will not be displayed publicly unless you allow it to, or just save the form as it is to never hear about it again.'
       subject: New hashtag up for review on %{instance} (#%{name})
+  aliases:
+    add_new: Create alias
+    created_msg: Successfully created a new alias. You can now initiate the move from the old account.
+    deleted_msg: Successfully remove the alias. Moving from that account to this one will no longer be possible.
+    hint_html: If you want to move from another account to this one, here you can create an alias, which is required before you can proceed with moving followers from the old account to this one. This action by itself is <strong>harmless and reversible</strong>. <strong>The account migration is initiated from the old account</strong>.
+    remove: Unlink alias
   appearance:
     advanced_web_interface: Advanced web interface
     advanced_web_interface_hint: 'If you want to make use of your entire screen width, the advanced web interface allows you to configure many different columns to see as much information at the same time as you want: Home, notifications, federated timeline, any number of lists and hashtags.'
@@ -613,6 +619,7 @@ en:
       confirming: Waiting for e-mail confirmation to be completed.
       functional: Your account is fully operational.
       pending: Your application is pending review by our staff. This may take some time. You will receive an e-mail if your application is approved.
+      redirecting_to: Your account is inactive because it is currently redirecting to %{acct}.
     trouble_logging_in: Trouble logging in?
   authorize_follow:
     already_following: You are already following this account
@@ -801,10 +808,32 @@ en:
       images_and_video: Cannot attach a video to a status that already contains images
       too_many: Cannot attach more than 4 files
   migrations:
-    acct: username@domain of the new account
-    currently_redirecting: 'Your profile is set to redirect to:'
-    proceed: Save
-    updated_msg: Your account migration setting successfully updated!
+    acct: Moved to
+    cancel: Cancel redirect
+    cancel_explanation: Cancelling the redirect will re-activate your current account, but will not bring back followers that have been moved to that account.
+    cancelled_msg: Successfully cancelled the redirect.
+    errors:
+      already_moved: is the same account you have already moved to
+      missing_also_known_as: is not back-referencing this account
+      move_to_self: cannot be current account
+      not_found: could not be found
+      on_cooldown: You are on cooldown
+    followers_count: Followers at time of move
+    incoming_migrations: Moving from a different account
+    incoming_migrations_html: To move from another account to this one, first you need to <a href="%{path}">create an account alias</a>.
+    moved_msg: Your account is now redirecting to %{acct} and your followers are being moved over.
+    not_redirecting: Your account is not redirecting to any other account currently.
+    on_cooldown: You have recently migrated your account. This function will become available again in %{count} days.
+    past_migrations: Past migrations
+    proceed_with_move: Move followers
+    redirecting_to: Your account is redirecting to %{acct}.
+    warning:
+      backreference_required: The new account must first be configured to back-reference this one
+      before: 'Before proceeding, please read these notes carefully:'
+      cooldown: After moving there is a cooldown period during which you will not be able to move again
+      disabled_account: Your current account will not be fully usable afterwards. However, you will have access to data export as well as re-activation.
+      followers: This action will move all followers from the current account to the new account
+      other_data: No other data will be moved automatically
   moderation:
     title: Moderation
   notification_mailer:
@@ -950,6 +979,7 @@ en:
   settings:
     account: Account
     account_settings: Account settings
+    aliases: Account aliases
     appearance: Appearance
     authorized_apps: Authorized apps
     back: Back to Mastodon
diff --git a/config/locales/simple_form.en.yml b/config/locales/simple_form.en.yml
index c9ffcfc13..3d909e999 100644
--- a/config/locales/simple_form.en.yml
+++ b/config/locales/simple_form.en.yml
@@ -2,6 +2,10 @@
 en:
   simple_form:
     hints:
+      account_alias:
+        acct: Specify the username@domain of the account you want to move from
+      account_migration:
+        acct: Specify the username@domain of the account you want to move to
       account_warning_preset:
         text: You can use toot syntax, such as URLs, hashtags and mentions
       admin_account_action:
@@ -15,6 +19,8 @@ en:
         avatar: PNG, GIF or JPG. At most %{size}. Will be downscaled to %{dimensions}px
         bot: This account mainly performs automated actions and might not be monitored
         context: One or multiple contexts where the filter should apply
+        current_password: For security purposes please enter the password of the current account
+        current_username: To confirm, please enter the username of the current account
         digest: Only sent after a long period of inactivity and only if you have received any personal messages in your absence
         discoverable: The profile directory is another way by which your account can reach a wider audience
         email: You will be sent a confirmation e-mail
@@ -60,6 +66,10 @@ en:
         fields:
           name: Label
           value: Content
+      account_alias:
+        acct: Handle of the old account
+      account_migration:
+        acct: Handle of the new account
       account_warning_preset:
         text: Preset text
       admin_account_action:
diff --git a/config/navigation.rb b/config/navigation.rb
index 38668bbf7..32c299143 100644
--- a/config/navigation.rb
+++ b/config/navigation.rb
@@ -5,7 +5,7 @@ SimpleNavigation::Configuration.run do |navigation|
     n.item :web, safe_join([fa_icon('chevron-left fw'), t('settings.back')]), root_url
 
     n.item :profile, safe_join([fa_icon('user fw'), t('settings.profile')]), settings_profile_url, if: -> { current_user.functional? } do |s|
-      s.item :profile, safe_join([fa_icon('pencil fw'), t('settings.appearance')]), settings_profile_url, highlights_on: %r{/settings/profile|/settings/migration}
+      s.item :profile, safe_join([fa_icon('pencil fw'), t('settings.appearance')]), settings_profile_url
       s.item :featured_tags, safe_join([fa_icon('hashtag fw'), t('settings.featured_tags')]), settings_featured_tags_url
       s.item :identity_proofs, safe_join([fa_icon('key fw'), t('settings.identity_proofs')]), settings_identity_proofs_path, highlights_on: %r{/settings/identity_proofs*}, if: proc { current_account.identity_proofs.exists? }
     end
@@ -20,13 +20,13 @@ SimpleNavigation::Configuration.run do |navigation|
     n.item :filters, safe_join([fa_icon('filter fw'), t('filters.index.title')]), filters_path, highlights_on: %r{/filters}, if: -> { current_user.functional? }
 
     n.item :security, safe_join([fa_icon('lock fw'), t('settings.account')]), edit_user_registration_url do |s|
-      s.item :password, safe_join([fa_icon('lock fw'), t('settings.account_settings')]), edit_user_registration_url, highlights_on: %r{/auth/edit|/settings/delete}
+      s.item :password, safe_join([fa_icon('lock fw'), t('settings.account_settings')]), edit_user_registration_url, highlights_on: %r{/auth/edit|/settings/delete|/settings/migration|/settings/aliases}
       s.item :two_factor_authentication, safe_join([fa_icon('mobile fw'), t('settings.two_factor_authentication')]), settings_two_factor_authentication_url, highlights_on: %r{/settings/two_factor_authentication}
       s.item :authorized_apps, safe_join([fa_icon('list fw'), t('settings.authorized_apps')]), oauth_authorized_applications_url
     end
 
-    n.item :data, safe_join([fa_icon('cloud-download fw'), t('settings.import_and_export')]), settings_export_url, if: -> { current_user.functional? } do |s|
-      s.item :import, safe_join([fa_icon('cloud-upload fw'), t('settings.import')]), settings_import_url
+    n.item :data, safe_join([fa_icon('cloud-download fw'), t('settings.import_and_export')]), settings_export_url do |s|
+      s.item :import, safe_join([fa_icon('cloud-upload fw'), t('settings.import')]), settings_import_url, if: -> { current_user.functional? }
       s.item :export, safe_join([fa_icon('cloud-download fw'), t('settings.export')]), settings_export_url
     end
 
diff --git a/config/routes.rb b/config/routes.rb
index dcfa079a0..37e0cbdee 100644
--- a/config/routes.rb
+++ b/config/routes.rb
@@ -134,8 +134,14 @@ Rails.application.routes.draw do
     end
 
     resource :delete, only: [:show, :destroy]
-    resource :migration, only: [:show, :update]
 
+    resource :migration, only: [:show, :create] do
+      collection do
+        post :cancel
+      end
+    end
+
+    resources :aliases, only: [:index, :create, :destroy]
     resources :sessions, only: [:destroy]
     resources :featured_tags, only: [:index, :create, :destroy]
   end
diff --git a/db/migrate/20190914202517_create_account_migrations.rb b/db/migrate/20190914202517_create_account_migrations.rb
new file mode 100644
index 000000000..cb9d71c09
--- /dev/null
+++ b/db/migrate/20190914202517_create_account_migrations.rb
@@ -0,0 +1,12 @@
+class CreateAccountMigrations < ActiveRecord::Migration[5.2]
+  def change
+    create_table :account_migrations do |t|
+      t.belongs_to :account, foreign_key: { on_delete: :cascade }
+      t.string :acct, null: false, default: ''
+      t.bigint :followers_count, null: false, default: 0
+      t.belongs_to :target_account, foreign_key: { to_table: :accounts, on_delete: :nullify }
+
+      t.timestamps
+    end
+  end
+end
diff --git a/db/migrate/20190915194355_create_account_aliases.rb b/db/migrate/20190915194355_create_account_aliases.rb
new file mode 100644
index 000000000..32ce031d9
--- /dev/null
+++ b/db/migrate/20190915194355_create_account_aliases.rb
@@ -0,0 +1,11 @@
+class CreateAccountAliases < ActiveRecord::Migration[5.2]
+  def change
+    create_table :account_aliases do |t|
+      t.belongs_to :account, foreign_key: { on_delete: :cascade }
+      t.string :acct, null: false, default: ''
+      t.string :uri, null: false, default: ''
+
+      t.timestamps
+    end
+  end
+end
diff --git a/db/schema.rb b/db/schema.rb
index 749f79dee..fabeb16f3 100644
--- a/db/schema.rb
+++ b/db/schema.rb
@@ -15,6 +15,15 @@ ActiveRecord::Schema.define(version: 2019_09_17_213523) do
   # These are extensions that must be enabled in order to support this database
   enable_extension "plpgsql"
 
+  create_table "account_aliases", force: :cascade do |t|
+    t.bigint "account_id"
+    t.string "acct", default: "", null: false
+    t.string "uri", default: "", null: false
+    t.datetime "created_at", null: false
+    t.datetime "updated_at", null: false
+    t.index ["account_id"], name: "index_account_aliases_on_account_id"
+  end
+
   create_table "account_conversations", force: :cascade do |t|
     t.bigint "account_id"
     t.bigint "conversation_id"
@@ -49,6 +58,17 @@ ActiveRecord::Schema.define(version: 2019_09_17_213523) do
     t.index ["account_id"], name: "index_account_identity_proofs_on_account_id"
   end
 
+  create_table "account_migrations", force: :cascade do |t|
+    t.bigint "account_id"
+    t.string "acct", default: "", null: false
+    t.bigint "followers_count", default: 0, null: false
+    t.bigint "target_account_id"
+    t.datetime "created_at", null: false
+    t.datetime "updated_at", null: false
+    t.index ["account_id"], name: "index_account_migrations_on_account_id"
+    t.index ["target_account_id"], name: "index_account_migrations_on_target_account_id"
+  end
+
   create_table "account_moderation_notes", force: :cascade do |t|
     t.text "content", null: false
     t.bigint "account_id", null: false
@@ -768,10 +788,13 @@ ActiveRecord::Schema.define(version: 2019_09_17_213523) do
     t.index ["user_id"], name: "index_web_settings_on_user_id", unique: true
   end
 
+  add_foreign_key "account_aliases", "accounts", on_delete: :cascade
   add_foreign_key "account_conversations", "accounts", on_delete: :cascade
   add_foreign_key "account_conversations", "conversations", on_delete: :cascade
   add_foreign_key "account_domain_blocks", "accounts", name: "fk_206c6029bd", on_delete: :cascade
   add_foreign_key "account_identity_proofs", "accounts", on_delete: :cascade
+  add_foreign_key "account_migrations", "accounts", column: "target_account_id", on_delete: :nullify
+  add_foreign_key "account_migrations", "accounts", on_delete: :cascade
   add_foreign_key "account_moderation_notes", "accounts"
   add_foreign_key "account_moderation_notes", "accounts", column: "target_account_id"
   add_foreign_key "account_pins", "accounts", column: "target_account_id", on_delete: :cascade
diff --git a/spec/controllers/settings/migrations_controller_spec.rb b/spec/controllers/settings/migrations_controller_spec.rb
index 4d814a45e..36e4ba86e 100644
--- a/spec/controllers/settings/migrations_controller_spec.rb
+++ b/spec/controllers/settings/migrations_controller_spec.rb
@@ -21,6 +21,7 @@ describe Settings::MigrationsController do
 
       let(:user) { Fabricate(:user, account: account) }
       let(:account) { Fabricate(:account, moved_to_account: moved_to_account) }
+
       before { sign_in user, scope: :user }
 
       context 'when user does not have moved to account' do
@@ -32,7 +33,7 @@ describe Settings::MigrationsController do
         end
       end
 
-      context 'when user does not have moved to account' do
+      context 'when user has a moved to account' do
         let(:moved_to_account) { Fabricate(:account) }
 
         it 'renders show page' do
@@ -43,21 +44,22 @@ describe Settings::MigrationsController do
     end
   end
 
-  describe 'PUT #update' do
+  describe 'POST #create' do
     context 'when user is not sign in' do
-      subject { put :update }
+      subject { post :create }
 
       it_behaves_like 'authenticate user'
     end
 
     context 'when user is sign in' do
-      subject { put :update, params: { migration: { acct: acct } } }
+      subject { post :create, params: { account_migration: { acct: acct, current_password: '12345678' } } }
+
+      let(:user) { Fabricate(:user, password: '12345678') }
 
-      let(:user) { Fabricate(:user) }
       before { sign_in user, scope: :user }
 
       context 'when migration account is changed' do
-        let(:acct) { Fabricate(:account) }
+        let(:acct) { Fabricate(:account, also_known_as: [ActivityPub::TagManager.instance.uri_for(user.account)]) }
 
         it 'updates moved to account' do
           is_expected.to redirect_to settings_migration_path
diff --git a/spec/fabricators/account_alias_fabricator.rb b/spec/fabricators/account_alias_fabricator.rb
new file mode 100644
index 000000000..94dde9bb8
--- /dev/null
+++ b/spec/fabricators/account_alias_fabricator.rb
@@ -0,0 +1,5 @@
+Fabricator(:account_alias) do
+  account
+  acct 'test@example.com'
+  uri 'https://example.com/users/test'
+end
diff --git a/spec/fabricators/account_migration_fabricator.rb b/spec/fabricators/account_migration_fabricator.rb
new file mode 100644
index 000000000..3b3fc2077
--- /dev/null
+++ b/spec/fabricators/account_migration_fabricator.rb
@@ -0,0 +1,6 @@
+Fabricator(:account_migration) do
+  account
+  target_account
+  followers_count 1234
+  acct 'test@example.com'
+end
diff --git a/spec/models/account_alias_spec.rb b/spec/models/account_alias_spec.rb
new file mode 100644
index 000000000..27ec215aa
--- /dev/null
+++ b/spec/models/account_alias_spec.rb
@@ -0,0 +1,5 @@
+require 'rails_helper'
+
+RSpec.describe AccountAlias, type: :model do
+
+end
diff --git a/spec/models/account_migration_spec.rb b/spec/models/account_migration_spec.rb
new file mode 100644
index 000000000..8461b4b28
--- /dev/null
+++ b/spec/models/account_migration_spec.rb
@@ -0,0 +1,5 @@
+require 'rails_helper'
+
+RSpec.describe AccountMigration, type: :model do
+
+end
-- 
cgit 


From f497d14b19ce150ee19e6478c9018833e28c7d52 Mon Sep 17 00:00:00 2001
From: Takeshi Umeda <noel.yoshiba@gmail.com>
Date: Sat, 21 Sep 2019 16:11:21 +0900
Subject: Addition of update activity distribution by alias, minor correction
 (#11905)

* Addition of update activity distribution by alias, minor correction

* Distribute Update activity after adding alias
* Add uniqueness verification to alias uri
* accept acct starting with @

* fix double-quoted to single-quoted
---
 app/controllers/settings/aliases_controller.rb | 1 +
 app/models/account_alias.rb                    | 6 ++++++
 2 files changed, 7 insertions(+)

(limited to 'app/models')

diff --git a/app/controllers/settings/aliases_controller.rb b/app/controllers/settings/aliases_controller.rb
index da0a4a9fa..b7c9a409d 100644
--- a/app/controllers/settings/aliases_controller.rb
+++ b/app/controllers/settings/aliases_controller.rb
@@ -15,6 +15,7 @@ class Settings::AliasesController < Settings::BaseController
     @alias = current_account.aliases.build(resource_params)
 
     if @alias.save
+      ActivityPub::UpdateDistributionWorker.perform_async(current_account.id)
       redirect_to settings_aliases_path, notice: I18n.t('aliases.created_msg')
     else
       render :index
diff --git a/app/models/account_alias.rb b/app/models/account_alias.rb
index e9a0dd79e..66f8ce409 100644
--- a/app/models/account_alias.rb
+++ b/app/models/account_alias.rb
@@ -17,11 +17,17 @@ class AccountAlias < ApplicationRecord
 
   validates :acct, presence: true, domain: { acct: true }
   validates :uri, presence: true
+  validates :uri, uniqueness: { scope: :account_id }
 
   before_validation :set_uri
   after_create :add_to_account
   after_destroy :remove_from_account
 
+  def acct=(val)
+    val = val.to_s.strip
+    super(val.start_with?('@') ? val[1..-1] : val)
+  end
+
   private
 
   def set_uri
-- 
cgit 


From b18aea91e3b7c89338c2b0cd07e037e6557514ee Mon Sep 17 00:00:00 2001
From: Takeshi Umeda <noel.yoshiba@gmail.com>
Date: Sat, 21 Sep 2019 16:11:38 +0900
Subject: Accept acct starting with @ in account migration (#11907)

---
 app/models/account_migration.rb | 5 +++++
 1 file changed, 5 insertions(+)

(limited to 'app/models')

diff --git a/app/models/account_migration.rb b/app/models/account_migration.rb
index 15830bffb..e2c2cb085 100644
--- a/app/models/account_migration.rb
+++ b/app/models/account_migration.rb
@@ -46,6 +46,11 @@ class AccountMigration < ApplicationRecord
     created_at + COOLDOWN_PERIOD
   end
 
+  def acct=(val)
+    val = val.to_s.strip
+    super(val.start_with?('@') ? val[1..-1] : val)
+  end
+
   private
 
   def set_target_account
-- 
cgit 


From b359974d9b356bb723fe046466b178328cf9bbaf Mon Sep 17 00:00:00 2001
From: ThibG <thib@sitedethib.com>
Date: Sun, 22 Sep 2019 14:15:18 +0200
Subject: Show user what options they have voted (#11195)

* Add own_votes field to poll results in REST API

Fixes #10679

* Display user votes in WebUI

* Update styling

* Add vote checkmark to public pages
---
 app/javascript/mastodon/actions/importer/normalizer.js |  3 ++-
 app/javascript/mastodon/components/poll.js             |  7 ++++++-
 app/javascript/styles/mastodon/polls.scss              | 10 ++++++++--
 app/models/poll.rb                                     |  4 ++++
 app/serializers/rest/poll_serializer.rb                |  5 +++++
 app/views/statuses/_poll.html.haml                     |  8 ++++++--
 6 files changed, 31 insertions(+), 6 deletions(-)

(limited to 'app/models')

diff --git a/app/javascript/mastodon/actions/importer/normalizer.js b/app/javascript/mastodon/actions/importer/normalizer.js
index 5e7e78e69..f7108fdb9 100644
--- a/app/javascript/mastodon/actions/importer/normalizer.js
+++ b/app/javascript/mastodon/actions/importer/normalizer.js
@@ -73,8 +73,9 @@ export function normalizePoll(poll) {
 
   const emojiMap = makeEmojiMap(normalPoll);
 
-  normalPoll.options = poll.options.map(option => ({
+  normalPoll.options = poll.options.map((option, index) => ({
     ...option,
+    voted: poll.own_votes && poll.own_votes.includes(index),
     title_emojified: emojify(escapeTextContentForBrowser(option.title), emojiMap),
   }));
 
diff --git a/app/javascript/mastodon/components/poll.js b/app/javascript/mastodon/components/poll.js
index 373f710d3..4c9b23b77 100644
--- a/app/javascript/mastodon/components/poll.js
+++ b/app/javascript/mastodon/components/poll.js
@@ -10,6 +10,7 @@ import spring from 'react-motion/lib/spring';
 import escapeTextContentForBrowser from 'escape-html';
 import emojify from 'mastodon/features/emoji/emoji';
 import RelativeTimestamp from './relative_timestamp';
+import Icon from 'mastodon/components/icon';
 
 const messages = defineMessages({
   closed: { id: 'poll.closed', defaultMessage: 'Closed' },
@@ -103,6 +104,7 @@ class Poll extends ImmutablePureComponent {
     const percent            = poll.get('votes_count') === 0 ? 0 : (option.get('votes_count') / poll.get('votes_count')) * 100;
     const leading            = poll.get('options').filterNot(other => other.get('title') === option.get('title')).every(other => option.get('votes_count') > other.get('votes_count'));
     const active             = !!this.state.selected[`${optionIndex}`];
+    const voted              = option.get('voted') || (poll.get('own_votes') && poll.get('own_votes').includes(optionIndex));
 
     let titleEmojified = option.get('title_emojified');
     if (!titleEmojified) {
@@ -131,7 +133,10 @@ class Poll extends ImmutablePureComponent {
           />
 
           {!showResults && <span className={classNames('poll__input', { checkbox: poll.get('multiple'), active })} />}
-          {showResults && <span className='poll__number'>{Math.round(percent)}%</span>}
+          {showResults && <span className='poll__number'>
+            {!!voted && <Icon id='check' className='poll__vote__mark' />}
+            {Math.round(percent)}%
+          </span>}
 
           <span dangerouslySetInnerHTML={{ __html: titleEmojified }} />
         </label>
diff --git a/app/javascript/styles/mastodon/polls.scss b/app/javascript/styles/mastodon/polls.scss
index e80220f27..85ba138b4 100644
--- a/app/javascript/styles/mastodon/polls.scss
+++ b/app/javascript/styles/mastodon/polls.scss
@@ -95,13 +95,19 @@
 
   &__number {
     display: inline-block;
-    width: 36px;
+    width: 48px;
     font-weight: 700;
     padding: 0 10px;
     text-align: right;
     margin-top: auto;
     margin-bottom: auto;
-    flex: 0 0 36px;
+    flex: 0 0 48px;
+  }
+
+  &__vote__mark {
+    float: left;
+    color: $valid-value-color;
+    line-height: 18px;
   }
 
   &__footer {
diff --git a/app/models/poll.rb b/app/models/poll.rb
index 8f72c7b11..55a8f13a6 100644
--- a/app/models/poll.rb
+++ b/app/models/poll.rb
@@ -54,6 +54,10 @@ class Poll < ApplicationRecord
     account.id == account_id || votes.where(account: account).exists?
   end
 
+  def own_votes(account)
+    votes.where(account: account).pluck(:choice)
+  end
+
   delegate :local?, to: :account
 
   def remote?
diff --git a/app/serializers/rest/poll_serializer.rb b/app/serializers/rest/poll_serializer.rb
index 356c45b83..eb98bb2d2 100644
--- a/app/serializers/rest/poll_serializer.rb
+++ b/app/serializers/rest/poll_serializer.rb
@@ -8,6 +8,7 @@ class REST::PollSerializer < ActiveModel::Serializer
   has_many :emojis, serializer: REST::CustomEmojiSerializer
 
   attribute :voted, if: :current_user?
+  attribute :own_votes, if: :current_user?
 
   def id
     object.id.to_s
@@ -21,6 +22,10 @@ class REST::PollSerializer < ActiveModel::Serializer
     object.voted?(current_user.account)
   end
 
+  def own_votes
+    object.own_votes(current_user.account)
+  end
+
   def current_user?
     !current_user.nil?
   end
diff --git a/app/views/statuses/_poll.html.haml b/app/views/statuses/_poll.html.haml
index ba34890df..d6b36a5d1 100644
--- a/app/views/statuses/_poll.html.haml
+++ b/app/views/statuses/_poll.html.haml
@@ -1,15 +1,19 @@
 - show_results = (user_signed_in? && poll.voted?(current_account)) || poll.expired?
+- own_votes = user_signed_in? ? poll.own_votes(current_account) : []
 
 .poll
   %ul
-    - poll.loaded_options.each do |option|
+    - poll.loaded_options.each_with_index do |option, index|
       %li
         - if show_results
           - percent = poll.votes_count > 0 ? 100 * option.votes_count / poll.votes_count : 0
           %span.poll__chart{ style: "width: #{percent}%" }
 
           %label.poll__text><
-            %span.poll__number= percent.round
+            %span.poll__number><
+              - if own_votes.include?(index)
+                %i.poll__vote__mark.fa.fa-check
+              = percent.round
             = Formatter.instance.format_poll_option(status, option, autoplay: autoplay)
         - else
           %label.poll__text><
-- 
cgit 


From a1f04c1e3497e9dff5970038461d9f454f2650df Mon Sep 17 00:00:00 2001
From: Eugen Rochko <eugen@zeonfederated.com>
Date: Tue, 24 Sep 2019 04:35:36 +0200
Subject: Fix authentication before 2FA challenge (#11943)

Regression from #11831
---
 app/controllers/auth/sessions_controller.rb   | 61 +++++++++++++++------------
 app/models/concerns/ldap_authenticable.rb     | 44 +++++++++++++++----
 config/application.rb                         |  3 +-
 config/initializers/devise.rb                 | 11 ++---
 lib/devise/ldap_authenticatable.rb            | 55 ------------------------
 lib/devise/two_factor_ldap_authenticatable.rb | 32 ++++++++++++++
 lib/devise/two_factor_pam_authenticatable.rb  | 31 ++++++++++++++
 7 files changed, 139 insertions(+), 98 deletions(-)
 delete mode 100644 lib/devise/ldap_authenticatable.rb
 create mode 100644 lib/devise/two_factor_ldap_authenticatable.rb
 create mode 100644 lib/devise/two_factor_pam_authenticatable.rb

(limited to 'app/models')

diff --git a/app/controllers/auth/sessions_controller.rb b/app/controllers/auth/sessions_controller.rb
index b3113bbef..f48b17c79 100644
--- a/app/controllers/auth/sessions_controller.rb
+++ b/app/controllers/auth/sessions_controller.rb
@@ -8,6 +8,8 @@ class Auth::SessionsController < Devise::SessionsController
   skip_before_action :require_no_authentication, only: [:create]
   skip_before_action :require_functional!
 
+  prepend_before_action :authenticate_with_two_factor, if: :two_factor_enabled?, only: [:create]
+
   before_action :set_instance_presenter, only: [:new]
   before_action :set_body_classes
 
@@ -20,22 +22,9 @@ class Auth::SessionsController < Devise::SessionsController
   end
 
   def create
-    self.resource = begin
-      if user_params[:email].blank? && session[:otp_user_id].present?
-        User.find(session[:otp_user_id])
-      else
-        warden.authenticate!(auth_options)
-      end
-    end
-
-    if resource.otp_required_for_login?
-      if user_params[:otp_attempt].present? && session[:otp_user_id].present?
-        authenticate_with_two_factor_via_otp(resource)
-      else
-        prompt_for_two_factor(resource)
-      end
-    else
-      authenticate_and_respond(resource)
+    super do |resource|
+      remember_me(resource)
+      flash.delete(:notice)
     end
   end
 
@@ -49,6 +38,16 @@ class Auth::SessionsController < Devise::SessionsController
 
   protected
 
+  def find_user
+    if session[:otp_user_id]
+      User.find(session[:otp_user_id])
+    else
+      user   = User.authenticate_with_ldap(user_params) if Devise.ldap_authentication
+      user ||= User.authenticate_with_pam(user_params) if Devise.pam_authentication
+      user ||= User.find_for_authentication(email: user_params[:email])
+    end
+  end
+
   def user_params
     params.require(:user).permit(:email, :password, :otp_attempt)
   end
@@ -71,6 +70,10 @@ class Auth::SessionsController < Devise::SessionsController
     super
   end
 
+  def two_factor_enabled?
+    find_user&.otp_required_for_login?
+  end
+
   def valid_otp_attempt?(user)
     user.validate_and_consume_otp!(user_params[:otp_attempt]) ||
       user.invalidate_otp_backup_code!(user_params[:otp_attempt])
@@ -78,10 +81,24 @@ class Auth::SessionsController < Devise::SessionsController
     false
   end
 
+  def authenticate_with_two_factor
+    user = self.resource = find_user
+
+    if user_params[:otp_attempt].present? && session[:otp_user_id]
+      authenticate_with_two_factor_via_otp(user)
+    elsif user.present? && (user.encrypted_password.blank? || user.valid_password?(user_params[:password]))
+      # If encrypted_password is blank, we got the user from LDAP or PAM,
+      # so credentials are already valid
+
+      prompt_for_two_factor(user)
+    end
+  end
+
   def authenticate_with_two_factor_via_otp(user)
     if valid_otp_attempt?(user)
       session.delete(:otp_user_id)
-      authenticate_and_respond(user)
+      remember_me(user)
+      sign_in(user)
     else
       flash.now[:alert] = I18n.t('users.invalid_otp_token')
       prompt_for_two_factor(user)
@@ -90,16 +107,10 @@ class Auth::SessionsController < Devise::SessionsController
 
   def prompt_for_two_factor(user)
     session[:otp_user_id] = user.id
+    @body_classes = 'lighter'
     render :two_factor
   end
 
-  def authenticate_and_respond(user)
-    sign_in(user)
-    remember_me(user)
-
-    respond_with user, location: after_sign_in_path_for(user)
-  end
-
   private
 
   def set_instance_presenter
@@ -112,11 +123,9 @@ class Auth::SessionsController < Devise::SessionsController
 
   def home_paths(resource)
     paths = [about_path]
-
     if single_user_mode? && resource.is_a?(User)
       paths << short_account_path(username: resource.account)
     end
-
     paths
   end
 
diff --git a/app/models/concerns/ldap_authenticable.rb b/app/models/concerns/ldap_authenticable.rb
index 84ff84c4b..117993947 100644
--- a/app/models/concerns/ldap_authenticable.rb
+++ b/app/models/concerns/ldap_authenticable.rb
@@ -3,24 +3,50 @@
 module LdapAuthenticable
   extend ActiveSupport::Concern
 
-  def ldap_setup(_attributes)
-    self.confirmed_at = Time.now.utc
-    self.admin        = false
-    self.external     = true
+  class_methods do
+    def authenticate_with_ldap(params = {})
+      ldap   = Net::LDAP.new(ldap_options)
+      filter = format(Devise.ldap_search_filter, uid: Devise.ldap_uid, email: params[:email])
 
-    save!
-  end
+      if (user_info = ldap.bind_as(base: Devise.ldap_base, filter: filter, password: params[:password]))
+        ldap_get_user(user_info.first)
+      end
+    end
 
-  class_methods do
     def ldap_get_user(attributes = {})
       resource = joins(:account).find_by(accounts: { username: attributes[Devise.ldap_uid.to_sym].first })
 
       if resource.blank?
-        resource = new(email: attributes[:mail].first, agreement: true, account_attributes: { username: attributes[Devise.ldap_uid.to_sym].first })
-        resource.ldap_setup(attributes)
+        resource = new(email: attributes[:mail].first, agreement: true, account_attributes: { username: attributes[Devise.ldap_uid.to_sym].first }, admin: false, external: true, confirmed_at: Time.now.utc)
+        resource.save!
       end
 
       resource
     end
+
+    def ldap_options
+      opts = {
+        host: Devise.ldap_host,
+        port: Devise.ldap_port,
+        base: Devise.ldap_base,
+
+        auth: {
+          method: :simple,
+          username: Devise.ldap_bind_dn,
+          password: Devise.ldap_password,
+        },
+
+        connect_timeout: 10,
+      }
+
+      if [:simple_tls, :start_tls].include?(Devise.ldap_method)
+        opts[:encryption] = {
+          method: Devise.ldap_method,
+          tls_options: OpenSSL::SSL::SSLContext::DEFAULT_PARAMS.tap { |options| options[:verify_mode] = OpenSSL::SSL::VERIFY_NONE if Devise.ldap_tls_no_verify },
+        }
+      end
+
+      opts
+    end
   end
 end
diff --git a/config/application.rb b/config/application.rb
index 5fd37120d..3ced81b8f 100644
--- a/config/application.rb
+++ b/config/application.rb
@@ -13,7 +13,8 @@ require_relative '../lib/paperclip/video_transcoder'
 require_relative '../lib/paperclip/type_corrector'
 require_relative '../lib/mastodon/snowflake'
 require_relative '../lib/mastodon/version'
-require_relative '../lib/devise/ldap_authenticatable'
+require_relative '../lib/devise/two_factor_ldap_authenticatable'
+require_relative '../lib/devise/two_factor_pam_authenticatable'
 
 Dotenv::Railtie.load
 
diff --git a/config/initializers/devise.rb b/config/initializers/devise.rb
index 311583820..fd9a5a8b9 100644
--- a/config/initializers/devise.rb
+++ b/config/initializers/devise.rb
@@ -71,13 +71,10 @@ end
 
 Devise.setup do |config|
   config.warden do |manager|
-    manager.default_strategies(scope: :user).unshift :database_authenticatable
-    manager.default_strategies(scope: :user).unshift :ldap_authenticatable if Devise.ldap_authentication
-    manager.default_strategies(scope: :user).unshift :pam_authenticatable  if Devise.pam_authentication
-
-    # We handle 2FA in our own sessions controller so this gets in the way
-    manager.default_strategies(scope: :user).delete :two_factor_backupable
-    manager.default_strategies(scope: :user).delete :two_factor_authenticatable
+    manager.default_strategies(scope: :user).unshift :two_factor_ldap_authenticatable if Devise.ldap_authentication
+    manager.default_strategies(scope: :user).unshift :two_factor_pam_authenticatable  if Devise.pam_authentication
+    manager.default_strategies(scope: :user).unshift :two_factor_authenticatable
+    manager.default_strategies(scope: :user).unshift :two_factor_backupable
   end
 
   # The secret key used by Devise. Devise uses this key to generate
diff --git a/lib/devise/ldap_authenticatable.rb b/lib/devise/ldap_authenticatable.rb
deleted file mode 100644
index 6903d468d..000000000
--- a/lib/devise/ldap_authenticatable.rb
+++ /dev/null
@@ -1,55 +0,0 @@
-# frozen_string_literal: true
-
-require 'net/ldap'
-require 'devise/strategies/authenticatable'
-
-module Devise
-  module Strategies
-    class LdapAuthenticatable < Authenticatable
-      def authenticate!
-        if params[:user]
-          ldap = Net::LDAP.new(
-            host: Devise.ldap_host,
-            port: Devise.ldap_port,
-            base: Devise.ldap_base,
-            encryption: {
-              method: Devise.ldap_method,
-              tls_options: tls_options,
-            },
-            auth: {
-              method: :simple,
-              username: Devise.ldap_bind_dn,
-              password: Devise.ldap_password,
-            },
-            connect_timeout: 10
-          )
-
-          filter = format(Devise.ldap_search_filter, uid: Devise.ldap_uid, email: email)
-
-          if (user_info = ldap.bind_as(base: Devise.ldap_base, filter: filter, password: password))
-            user = User.ldap_get_user(user_info.first)
-            success!(user)
-          else
-            return fail(:invalid)
-          end
-        end
-      end
-
-      def email
-        params[:user][:email]
-      end
-
-      def password
-        params[:user][:password]
-      end
-
-      def tls_options
-        OpenSSL::SSL::SSLContext::DEFAULT_PARAMS.tap do |options|
-          options[:verify_mode] = OpenSSL::SSL::VERIFY_NONE if Devise.ldap_tls_no_verify
-        end
-      end
-    end
-  end
-end
-
-Warden::Strategies.add(:ldap_authenticatable, Devise::Strategies::LdapAuthenticatable)
diff --git a/lib/devise/two_factor_ldap_authenticatable.rb b/lib/devise/two_factor_ldap_authenticatable.rb
new file mode 100644
index 000000000..065aa2de8
--- /dev/null
+++ b/lib/devise/two_factor_ldap_authenticatable.rb
@@ -0,0 +1,32 @@
+# frozen_string_literal: true
+
+require 'net/ldap'
+require 'devise/strategies/base'
+
+module Devise
+  module Strategies
+    class TwoFactorLdapAuthenticatable < Base
+      def valid?
+        valid_params? && mapping.to.respond_to?(:authenticate_with_ldap)
+      end
+
+      def authenticate!
+        resource = mapping.to.authenticate_with_ldap(params[scope])
+
+        if resource && !resource.otp_required_for_login?
+          success!(resource)
+        else
+          fail(:invalid)
+        end
+      end
+
+      protected
+
+      def valid_params?
+        params[scope] && params[scope][:password].present?
+      end
+    end
+  end
+end
+
+Warden::Strategies.add(:two_factor_ldap_authenticatable, Devise::Strategies::TwoFactorLdapAuthenticatable)
diff --git a/lib/devise/two_factor_pam_authenticatable.rb b/lib/devise/two_factor_pam_authenticatable.rb
new file mode 100644
index 000000000..5ce723b33
--- /dev/null
+++ b/lib/devise/two_factor_pam_authenticatable.rb
@@ -0,0 +1,31 @@
+# frozen_string_literal: true
+
+require 'devise/strategies/base'
+
+module Devise
+  module Strategies
+    class TwoFactorPamAuthenticatable < Base
+      def valid?
+        valid_params? && mapping.to.respond_to?(:authenticate_with_pam)
+      end
+
+      def authenticate!
+        resource = mapping.to.authenticate_with_pam(params[scope])
+
+        if resource && !resource.otp_required_for_login?
+          success!(resource)
+        else
+          fail(:invalid)
+        end
+      end
+
+      protected
+
+      def valid_params?
+        params[scope] && params[scope][:password].present?
+      end
+    end
+  end
+end
+
+Warden::Strategies.add(:two_factor_pam_authenticatable, Devise::Strategies::TwoFactorPamAuthenticatable)
-- 
cgit 


From add4d4118c33562cf196f2045d6ce3aa309a40a0 Mon Sep 17 00:00:00 2001
From: Eugen Rochko <eugen@zeonfederated.com>
Date: Fri, 27 Sep 2019 02:13:34 +0200
Subject: Fix relays UI being available in whitelist/secure mode (#11963)

Fix relays UI referencing relay that is not functional
---
 app/controllers/admin/relays_controller.rb | 7 ++++++-
 app/models/relay.rb                        | 5 +----
 config/locales/en.yml                      | 3 ++-
 config/navigation.rb                       | 2 +-
 4 files changed, 10 insertions(+), 7 deletions(-)

(limited to 'app/models')

diff --git a/app/controllers/admin/relays_controller.rb b/app/controllers/admin/relays_controller.rb
index 1b02d3c36..6fbb6e063 100644
--- a/app/controllers/admin/relays_controller.rb
+++ b/app/controllers/admin/relays_controller.rb
@@ -3,6 +3,7 @@
 module Admin
   class RelaysController < BaseController
     before_action :set_relay, except: [:index, :new, :create]
+    before_action :require_signatures_enabled!, only: [:new, :create, :enable]
 
     def index
       authorize :relay, :update?
@@ -11,7 +12,7 @@ module Admin
 
     def new
       authorize :relay, :update?
-      @relay = Relay.new(inbox_url: Relay::PRESET_RELAY)
+      @relay = Relay.new
     end
 
     def create
@@ -54,5 +55,9 @@ module Admin
     def resource_params
       params.require(:relay).permit(:inbox_url)
     end
+
+    def require_signatures_enabled!
+      redirect_to admin_relays_path, alert: I18n.t('admin.relays.signatures_not_enabled') if authorized_fetch_mode?
+    end
   end
 end
diff --git a/app/models/relay.rb b/app/models/relay.rb
index 6934a5c62..8c8a97db3 100644
--- a/app/models/relay.rb
+++ b/app/models/relay.rb
@@ -12,8 +12,6 @@
 #
 
 class Relay < ApplicationRecord
-  PRESET_RELAY = 'https://relay.joinmastodon.org/inbox'
-
   validates :inbox_url, presence: true, uniqueness: true, url: true, if: :will_save_change_to_inbox_url?
 
   enum state: [:idle, :pending, :accepted, :rejected]
@@ -74,7 +72,6 @@ class Relay < ApplicationRecord
   end
 
   def ensure_disabled
-    return unless enabled?
-    disable!
+    disable! if enabled?
   end
 end
diff --git a/config/locales/en.yml b/config/locales/en.yml
index c29c7f871..c580c5ed5 100644
--- a/config/locales/en.yml
+++ b/config/locales/en.yml
@@ -257,7 +257,7 @@ en:
       updated_msg: Emoji successfully updated!
       upload: Upload
     dashboard:
-      authorized_fetch_mode: Authorized fetch mode
+      authorized_fetch_mode: Secure mode
       backlog: backlogged jobs
       config: Configuration
       feature_deletions: Account deletions
@@ -383,6 +383,7 @@ en:
       pending: Waiting for relay's approval
       save_and_enable: Save and enable
       setup: Setup a relay connection
+      signatures_not_enabled: Relays will not work correctly while secure mode or whitelist mode is enabled
       status: Status
       title: Relays
     report_notes:
diff --git a/config/navigation.rb b/config/navigation.rb
index 32c299143..eebd4f75e 100644
--- a/config/navigation.rb
+++ b/config/navigation.rb
@@ -47,7 +47,7 @@ SimpleNavigation::Configuration.run do |navigation|
       s.item :dashboard, safe_join([fa_icon('tachometer fw'), t('admin.dashboard.title')]), admin_dashboard_url
       s.item :settings, safe_join([fa_icon('cogs fw'), t('admin.settings.title')]), edit_admin_settings_url, if: -> { current_user.admin? }, highlights_on: %r{/admin/settings}
       s.item :custom_emojis, safe_join([fa_icon('smile-o fw'), t('admin.custom_emojis.title')]), admin_custom_emojis_url, highlights_on: %r{/admin/custom_emojis}
-      s.item :relays, safe_join([fa_icon('exchange fw'), t('admin.relays.title')]), admin_relays_url, if: -> { current_user.admin? }, highlights_on: %r{/admin/relays}
+      s.item :relays, safe_join([fa_icon('exchange fw'), t('admin.relays.title')]), admin_relays_url, if: -> { current_user.admin? && !whitelist_mode? }, highlights_on: %r{/admin/relays}
       s.item :sidekiq, safe_join([fa_icon('diamond fw'), 'Sidekiq']), sidekiq_url, link_html: { target: 'sidekiq' }, if: -> { current_user.admin? }
       s.item :pghero, safe_join([fa_icon('database fw'), 'PgHero']), pghero_url, link_html: { target: 'pghero' }, if: -> { current_user.admin? }
     end
-- 
cgit 


From 07b057eabb9cc923aa7fc6bb851084af048ed5d2 Mon Sep 17 00:00:00 2001
From: abcang <abcang1015@gmail.com>
Date: Fri, 27 Sep 2019 22:24:13 +0900
Subject: Validate Web::PushSubscription (#11971)

---
 app/models/web/push_subscription.rb                    |  4 ++++
 ...90927124642_remove_invalid_web_push_subscription.rb | 18 ++++++++++++++++++
 db/schema.rb                                           |  2 +-
 3 files changed, 23 insertions(+), 1 deletion(-)
 create mode 100644 db/post_migrate/20190927124642_remove_invalid_web_push_subscription.rb

(limited to 'app/models')

diff --git a/app/models/web/push_subscription.rb b/app/models/web/push_subscription.rb
index b57807d1c..c5dbb58ba 100644
--- a/app/models/web/push_subscription.rb
+++ b/app/models/web/push_subscription.rb
@@ -20,6 +20,10 @@ class Web::PushSubscription < ApplicationRecord
 
   has_one :session_activation, foreign_key: 'web_push_subscription_id', inverse_of: :web_push_subscription
 
+  validates :endpoint, presence: true
+  validates :key_p256dh, presence: true
+  validates :key_auth, presence: true
+
   def push(notification)
     I18n.with_locale(associated_user&.locale || I18n.default_locale) do
       push_payload(payload_for_notification(notification), 48.hours.seconds)
diff --git a/db/post_migrate/20190927124642_remove_invalid_web_push_subscription.rb b/db/post_migrate/20190927124642_remove_invalid_web_push_subscription.rb
new file mode 100644
index 000000000..c2397476a
--- /dev/null
+++ b/db/post_migrate/20190927124642_remove_invalid_web_push_subscription.rb
@@ -0,0 +1,18 @@
+# frozen_string_literal: true
+
+class RemoveInvalidWebPushSubscription < ActiveRecord::Migration[5.2]
+  disable_ddl_transaction!
+
+  def up
+    invalid_web_push_subscriptions = Web::PushSubscription.where(endpoint: '')
+                                                          .or(Web::PushSubscription.where(key_p256dh: ''))
+                                                          .or(Web::PushSubscription.where(key_auth: ''))
+                                                          .preload(:session_activation)
+    invalid_web_push_subscriptions.find_each do |web_push_subscription|
+      web_push_subscription.session_activation&.update!(web_push_subscription_id: nil)
+      web_push_subscription.destroy!
+    end
+  end
+
+  def down; end
+end
diff --git a/db/schema.rb b/db/schema.rb
index fabeb16f3..8eeaf48a0 100644
--- a/db/schema.rb
+++ b/db/schema.rb
@@ -10,7 +10,7 @@
 #
 # It's strongly recommended that you check this file into your version control system.
 
-ActiveRecord::Schema.define(version: 2019_09_17_213523) do
+ActiveRecord::Schema.define(version: 2019_09_27_124642) do
 
   # These are extensions that must be enabled in order to support this database
   enable_extension "plpgsql"
-- 
cgit 


From ab33c4df942ec3fdc4d891f3db7ac8cdd3436945 Mon Sep 17 00:00:00 2001
From: Eugen Rochko <eugen@zeonfederated.com>
Date: Sat, 28 Sep 2019 01:02:21 +0200
Subject: Add `exclude_unreviewed` param to `GET /api/v2/search` REST API
 (#11977)

Make it so normal search returns even unreviewed matches, but
autosuggestions do not.

Fix #11960
---
 app/controllers/api/v2/search_controller.rb |  2 +-
 app/javascript/mastodon/actions/compose.js  |  1 +
 app/models/tag.rb                           | 13 ++++++-------
 app/services/search_service.rb              |  3 ++-
 app/services/tag_search_service.rb          | 16 ++++++++++------
 spec/services/search_service_spec.rb        |  4 ++--
 6 files changed, 22 insertions(+), 17 deletions(-)

(limited to 'app/models')

diff --git a/app/controllers/api/v2/search_controller.rb b/app/controllers/api/v2/search_controller.rb
index c14cd22d7..cbd9b551d 100644
--- a/app/controllers/api/v2/search_controller.rb
+++ b/app/controllers/api/v2/search_controller.rb
@@ -22,7 +22,7 @@ class Api::V2::SearchController < Api::BaseController
       params[:q],
       current_account,
       limit_param(RESULTS_LIMIT),
-      search_params.merge(resolve: truthy_param?(:resolve))
+      search_params.merge(resolve: truthy_param?(:resolve), exclude_unreviewed: truthy_param?(:exclude_unreviewed))
     )
   end
 
diff --git a/app/javascript/mastodon/actions/compose.js b/app/javascript/mastodon/actions/compose.js
index 79d64dc3e..8e7906c73 100644
--- a/app/javascript/mastodon/actions/compose.js
+++ b/app/javascript/mastodon/actions/compose.js
@@ -369,6 +369,7 @@ const fetchComposeSuggestionsTags = throttle((dispatch, getState, token) => {
       q: token.slice(1),
       resolve: false,
       limit: 4,
+      exclude_unreviewed: true,
     },
   }).then(({ data }) => {
     dispatch(readyComposeSuggestionsTags(token, data.hashtags));
diff --git a/app/models/tag.rb b/app/models/tag.rb
index b52b9bc9f..9aca3983f 100644
--- a/app/models/tag.rb
+++ b/app/models/tag.rb
@@ -124,16 +124,15 @@ class Tag < ApplicationRecord
       end
     end
 
-    def search_for(term, limit = 5, offset = 0)
+    def search_for(term, limit = 5, offset = 0, options = {})
       normalized_term = normalize(term.strip).mb_chars.downcase.to_s
       pattern         = sanitize_sql_like(normalized_term) + '%'
+      query           = Tag.listable.where(arel_table[:name].lower.matches(pattern))
+      query           = query.where(arel_table[:name].lower.eq(normalized_term).or(arel_table[:reviewed_at].not_eq(nil))) if options[:exclude_unreviewed]
 
-      Tag.listable
-         .where(arel_table[:name].lower.matches(pattern))
-         .where(arel_table[:name].lower.eq(normalized_term).or(arel_table[:reviewed_at].not_eq(nil)))
-         .order(Arel.sql('length(name) ASC, name ASC'))
-         .limit(limit)
-         .offset(offset)
+      query.order(Arel.sql('length(name) ASC, name ASC'))
+           .limit(limit)
+           .offset(offset)
     end
 
     def find_normalized(name)
diff --git a/app/services/search_service.rb b/app/services/search_service.rb
index a5ba5dd11..3a498dcf4 100644
--- a/app/services/search_service.rb
+++ b/app/services/search_service.rb
@@ -60,7 +60,8 @@ class SearchService < BaseService
     TagSearchService.new.call(
       @query,
       limit: @limit,
-      offset: @offset
+      offset: @offset,
+      exclude_unreviewed: @options[:exclude_unreviewed]
     )
   end
 
diff --git a/app/services/tag_search_service.rb b/app/services/tag_search_service.rb
index 47b0e876e..b78d65625 100644
--- a/app/services/tag_search_service.rb
+++ b/app/services/tag_search_service.rb
@@ -2,11 +2,12 @@
 
 class TagSearchService < BaseService
   def call(query, options = {})
-    @query  = query.strip.gsub(/\A#/, '')
-    @offset = options[:offset].to_i
-    @limit  = options[:limit].to_i
+    @query   = query.strip.gsub(/\A#/, '')
+    @offset  = options.delete(:offset).to_i
+    @limit   = options.delete(:limit).to_i
+    @options = options
 
-    results = from_elasticsearch if Chewy.enabled?
+    results   = from_elasticsearch if Chewy.enabled?
     results ||= from_database
 
     results
@@ -72,12 +73,15 @@ class TagSearchService < BaseService
       },
     }
 
-    TagsIndex.query(query).filter(filter).limit(@limit).offset(@offset).objects.compact
+    definition = TagsIndex.query(query)
+    definition = definition.filter(filter) if @options[:exclude_unreviewed]
+
+    definition.limit(@limit).offset(@offset).objects.compact
   rescue Faraday::ConnectionFailed, Parslet::ParseFailed
     nil
   end
 
   def from_database
-    Tag.search_for(@query, @limit, @offset)
+    Tag.search_for(@query, @limit, @offset, @options)
   end
 end
diff --git a/spec/services/search_service_spec.rb b/spec/services/search_service_spec.rb
index ade306ed2..739bb9cf5 100644
--- a/spec/services/search_service_spec.rb
+++ b/spec/services/search_service_spec.rb
@@ -77,10 +77,10 @@ describe SearchService, type: :service do
         it 'includes the tag in the results' do
           query = '#tag'
           tag = Tag.new
-          allow(Tag).to receive(:search_for).with('tag', 10, 0).and_return([tag])
+          allow(Tag).to receive(:search_for).with('tag', 10, 0, exclude_unreviewed: nil).and_return([tag])
 
           results = subject.call(query, nil, 10)
-          expect(Tag).to have_received(:search_for).with('tag', 10, 0)
+          expect(Tag).to have_received(:search_for).with('tag', 10, 0, exclude_unreviewed: nil)
           expect(results).to eq empty_results.merge(hashtags: [tag])
         end
         it 'does not include tag when starts with @ character' do
-- 
cgit 


From 32ff78f7496012d9fca5b1ad13cddb650184b5ec Mon Sep 17 00:00:00 2001
From: Eugen Rochko <eugen@zeonfederated.com>
Date: Sat, 28 Sep 2019 01:33:02 +0200
Subject: Fix index not being used in Status.reblogs_map (#11982)

Regression from #11623
---
 app/models/status.rb | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

(limited to 'app/models')

diff --git a/app/models/status.rb b/app/models/status.rb
index 471bb03b4..6afaa14d1 100644
--- a/app/models/status.rb
+++ b/app/models/status.rb
@@ -305,7 +305,7 @@ class Status < ApplicationRecord
     end
 
     def reblogs_map(status_ids, account_id)
-      select('reblog_of_id').where(reblog_of_id: status_ids).where(account_id: account_id).reorder(nil).each_with_object({}) { |s, h| h[s.reblog_of_id] = true }
+      select('reblog_of_id').where(reblog_of_id: status_ids).where(account_id: account_id).unscoped.each_with_object({}) { |s, h| h[s.reblog_of_id] = true }
     end
 
     def mutes_map(conversation_ids, account_id)
-- 
cgit 


From 3ec80c7aec4c95c64c4cafb511610eab5fa31b1e Mon Sep 17 00:00:00 2001
From: Eugen Rochko <eugen@zeonfederated.com>
Date: Sat, 28 Sep 2019 01:33:16 +0200
Subject: Fix preview card image not being re-fetched even if link is re-posted
 (#11981)

Fix #11956
---
 app/models/preview_card.rb              | 4 ++++
 app/services/fetch_link_card_service.rb | 2 +-
 2 files changed, 5 insertions(+), 1 deletion(-)

(limited to 'app/models')

diff --git a/app/models/preview_card.rb b/app/models/preview_card.rb
index 9d6c1938a..4e89fbf85 100644
--- a/app/models/preview_card.rb
+++ b/app/models/preview_card.rb
@@ -47,6 +47,10 @@ class PreviewCard < ApplicationRecord
 
   before_save :extract_dimensions, if: :link?
 
+  def missing_image?
+    width.present? && height.present? && image_file_name.blank?
+  end
+
   def save_with_optional_image!
     save!
   rescue ActiveRecord::RecordInvalid
diff --git a/app/services/fetch_link_card_service.rb b/app/services/fetch_link_card_service.rb
index 4e75c370f..ac5503d46 100644
--- a/app/services/fetch_link_card_service.rb
+++ b/app/services/fetch_link_card_service.rb
@@ -22,7 +22,7 @@ class FetchLinkCardService < BaseService
     RedisLock.acquire(lock_options) do |lock|
       if lock.acquired?
         @card = PreviewCard.find_by(url: @url)
-        process_url if @card.nil? || @card.updated_at <= 2.weeks.ago
+        process_url if @card.nil? || @card.updated_at <= 2.weeks.ago || @card.missing_image?
       else
         raise Mastodon::RaceConditionError
       end
-- 
cgit 


From 50af41a00daaa99150b0fa92be0c9bde44ced273 Mon Sep 17 00:00:00 2001
From: Eugen Rochko <eugen@zeonfederated.com>
Date: Sat, 28 Sep 2019 05:23:32 +0200
Subject: Fix unscoped being used in the wrong place (#11987)

Regression from #11982
---
 app/models/status.rb | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

(limited to 'app/models')

diff --git a/app/models/status.rb b/app/models/status.rb
index 6afaa14d1..5e7474577 100644
--- a/app/models/status.rb
+++ b/app/models/status.rb
@@ -305,7 +305,7 @@ class Status < ApplicationRecord
     end
 
     def reblogs_map(status_ids, account_id)
-      select('reblog_of_id').where(reblog_of_id: status_ids).where(account_id: account_id).unscoped.each_with_object({}) { |s, h| h[s.reblog_of_id] = true }
+      unscoped.select('reblog_of_id').where(reblog_of_id: status_ids).where(account_id: account_id).each_with_object({}) { |s, h| h[s.reblog_of_id] = true }
     end
 
     def mutes_map(conversation_ids, account_id)
-- 
cgit 


From 163ed91af381d86bb6c52546c983effa4c9a18c3 Mon Sep 17 00:00:00 2001
From: Eugen Rochko <eugen@zeonfederated.com>
Date: Sun, 29 Sep 2019 05:03:19 +0200
Subject: Add (back) option to set redirect notice on account without moving
 followers (#11994)

Fix #11913
---
 .../settings/migration/redirects_controller.rb     | 45 +++++++++++++++++++++
 app/controllers/settings/migrations_controller.rb  |  9 -----
 app/models/account_migration.rb                    |  3 +-
 app/models/form/redirect.rb                        | 47 ++++++++++++++++++++++
 app/views/auth/registrations/edit.html.haml        | 11 +++++
 .../settings/migration/redirects/new.html.haml     | 27 +++++++++++++
 app/views/settings/migrations/show.html.haml       | 10 +++--
 config/locales/en.yml                              |  3 ++
 config/routes.rb                                   |  7 ++--
 9 files changed, 144 insertions(+), 18 deletions(-)
 create mode 100644 app/controllers/settings/migration/redirects_controller.rb
 create mode 100644 app/models/form/redirect.rb
 create mode 100644 app/views/settings/migration/redirects/new.html.haml

(limited to 'app/models')

diff --git a/app/controllers/settings/migration/redirects_controller.rb b/app/controllers/settings/migration/redirects_controller.rb
new file mode 100644
index 000000000..6e5b72ffb
--- /dev/null
+++ b/app/controllers/settings/migration/redirects_controller.rb
@@ -0,0 +1,45 @@
+# frozen_string_literal: true
+
+class Settings::Migration::RedirectsController < Settings::BaseController
+  layout 'admin'
+
+  before_action :authenticate_user!
+  before_action :require_not_suspended!
+
+  skip_before_action :require_functional!
+
+  def new
+    @redirect = Form::Redirect.new
+  end
+
+  def create
+    @redirect = Form::Redirect.new(resource_params.merge(account: current_account))
+
+    if @redirect.valid_with_challenge?(current_user)
+      current_account.update!(moved_to_account: @redirect.target_account)
+      ActivityPub::UpdateDistributionWorker.perform_async(current_account.id)
+      redirect_to settings_migration_path, notice: I18n.t('migrations.moved_msg', acct: current_account.moved_to_account.acct)
+    else
+      render :new
+    end
+  end
+
+  def destroy
+    if current_account.moved_to_account_id.present?
+      current_account.update!(moved_to_account: nil)
+      ActivityPub::UpdateDistributionWorker.perform_async(current_account.id)
+    end
+
+    redirect_to settings_migration_path, notice: I18n.t('migrations.cancelled_msg')
+  end
+
+  private
+
+  def resource_params
+    params.require(:form_redirect).permit(:acct, :current_password, :current_username)
+  end
+
+  def require_not_suspended!
+    forbidden if current_account.suspended?
+  end
+end
diff --git a/app/controllers/settings/migrations_controller.rb b/app/controllers/settings/migrations_controller.rb
index 90092c692..00bde1d61 100644
--- a/app/controllers/settings/migrations_controller.rb
+++ b/app/controllers/settings/migrations_controller.rb
@@ -27,15 +27,6 @@ class Settings::MigrationsController < Settings::BaseController
     end
   end
 
-  def cancel
-    if current_account.moved_to_account_id.present?
-      current_account.update!(moved_to_account: nil)
-      ActivityPub::UpdateDistributionWorker.perform_async(current_account.id)
-    end
-
-    redirect_to settings_migration_path, notice: I18n.t('migrations.cancelled_msg')
-  end
-
   helper_method :on_cooldown?
 
   private
diff --git a/app/models/account_migration.rb b/app/models/account_migration.rb
index e2c2cb085..681b5b2cd 100644
--- a/app/models/account_migration.rb
+++ b/app/models/account_migration.rb
@@ -47,8 +47,7 @@ class AccountMigration < ApplicationRecord
   end
 
   def acct=(val)
-    val = val.to_s.strip
-    super(val.start_with?('@') ? val[1..-1] : val)
+    super(val.to_s.strip.gsub(/\A@/, ''))
   end
 
   private
diff --git a/app/models/form/redirect.rb b/app/models/form/redirect.rb
new file mode 100644
index 000000000..a7961f8e8
--- /dev/null
+++ b/app/models/form/redirect.rb
@@ -0,0 +1,47 @@
+# frozen_string_literal: true
+
+class Form::Redirect
+  include ActiveModel::Model
+
+  attr_accessor :account, :target_account, :current_password,
+                :current_username
+
+  attr_reader :acct
+
+  validates :acct, presence: true, domain: { acct: true }
+  validate :validate_target_account
+
+  def valid_with_challenge?(current_user)
+    if current_user.encrypted_password.present?
+      errors.add(:current_password, :invalid) unless current_user.valid_password?(current_password)
+    else
+      errors.add(:current_username, :invalid) unless account.username == current_username
+    end
+
+    return false unless errors.empty?
+
+    set_target_account
+    valid?
+  end
+
+  def acct=(val)
+    @acct = val.to_s.strip.gsub(/\A@/, '')
+  end
+
+  private
+
+  def set_target_account
+    @target_account = ResolveAccountService.new.call(acct)
+  rescue Goldfinger::Error, HTTP::Error, OpenSSL::SSL::SSLError, Mastodon::Error
+    # Validation will take care of it
+  end
+
+  def validate_target_account
+    if target_account.nil?
+      errors.add(:acct, I18n.t('migrations.errors.not_found'))
+    else
+      errors.add(:acct, I18n.t('migrations.errors.already_moved')) if account.moved_to_account_id.present? && account.moved_to_account_id == target_account.id
+      errors.add(:acct, I18n.t('migrations.errors.move_to_self')) if account.id == target_account.id
+    end
+  end
+end
diff --git a/app/views/auth/registrations/edit.html.haml b/app/views/auth/registrations/edit.html.haml
index 885171c58..a155c75c9 100644
--- a/app/views/auth/registrations/edit.html.haml
+++ b/app/views/auth/registrations/edit.html.haml
@@ -30,7 +30,18 @@
 
 = render 'sessions'
 
+%hr.spacer/
+
+%h3= t('auth.migrate_account')
+%p.muted-hint= t('auth.migrate_account_html', path: settings_migration_path)
+
+%hr.spacer/
+
+%h3= t('migrations.incoming_migrations')
+%p.muted-hint= t('migrations.incoming_migrations_html', path: settings_aliases_path)
+
 - if open_deletion? && !current_account.suspended?
   %hr.spacer/
+
   %h3= t('auth.delete_account')
   %p.muted-hint= t('auth.delete_account_html', path: settings_delete_path)
diff --git a/app/views/settings/migration/redirects/new.html.haml b/app/views/settings/migration/redirects/new.html.haml
new file mode 100644
index 000000000..017450f4b
--- /dev/null
+++ b/app/views/settings/migration/redirects/new.html.haml
@@ -0,0 +1,27 @@
+- content_for :page_title do
+  = t('settings.migrate')
+
+= simple_form_for @redirect, url: settings_migration_redirect_path do |f|
+  %p.hint= t('migrations.warning.before')
+
+  %ul.hint
+    %li.warning-hint= t('migrations.warning.redirect')
+    %li.warning-hint= t('migrations.warning.other_data')
+    %li.warning-hint= t('migrations.warning.disabled_account')
+
+  %hr.spacer/
+
+  = render 'shared/error_messages', object: @redirect
+
+  .fields-row
+    .fields-row__column.fields-group.fields-row__column-6
+      = f.input :acct, wrapper: :with_block_label, input_html: { autocapitalize: 'none', autocorrect: 'off' }, label: t('simple_form.labels.account_migration.acct'), hint: t('simple_form.hints.account_migration.acct')
+
+    .fields-row__column.fields-group.fields-row__column-6
+      - if current_user.encrypted_password.present?
+        = f.input :current_password, wrapper: :with_block_label, input_html: { :autocomplete => 'off' }, required: true
+      - else
+        = f.input :current_username, wrapper: :with_block_label, input_html: { :autocomplete => 'off' }, required: true
+
+  .actions
+    = f.button :button, t('migrations.set_redirect'), type: :submit, class: 'button button--destructive'
diff --git a/app/views/settings/migrations/show.html.haml b/app/views/settings/migrations/show.html.haml
index 1e5c47726..078eaebc6 100644
--- a/app/views/settings/migrations/show.html.haml
+++ b/app/views/settings/migrations/show.html.haml
@@ -12,28 +12,32 @@
 
         %p.hint= t('migrations.cancel_explanation')
 
-        %p.hint= link_to t('migrations.cancel'), cancel_settings_migration_path, data: { method: :post }
+        %p.hint= link_to t('migrations.cancel'), settings_migration_redirect_path, data: { method: :delete }
   - else
     %p.hint
       %span.positive-hint= t('migrations.not_redirecting')
 
 %hr.spacer/
 
-%h3= t 'migrations.proceed_with_move'
+%h3= t('auth.migrate_account')
 
 = simple_form_for @migration, url: settings_migration_path do |f|
   - if on_cooldown?
-    %span.warning-hint= t('migrations.on_cooldown', count: ((@cooldown.cooldown_at - Time.now.utc) / 1.day.seconds).ceil)
+    %p.hint
+      %span.warning-hint= t('migrations.on_cooldown', count: ((@cooldown.cooldown_at - Time.now.utc) / 1.day.seconds).ceil)
   - else
     %p.hint= t('migrations.warning.before')
 
     %ul.hint
       %li.warning-hint= t('migrations.warning.followers')
+      %li.warning-hint= t('migrations.warning.redirect')
       %li.warning-hint= t('migrations.warning.other_data')
       %li.warning-hint= t('migrations.warning.backreference_required')
       %li.warning-hint= t('migrations.warning.cooldown')
       %li.warning-hint= t('migrations.warning.disabled_account')
 
+  %p.hint= t('migrations.warning.only_redirect_html', path: new_settings_migration_redirect_path)
+
   %hr.spacer/
 
   = render 'shared/error_messages', object: @migration
diff --git a/config/locales/en.yml b/config/locales/en.yml
index ee798e87f..1e7d0701b 100644
--- a/config/locales/en.yml
+++ b/config/locales/en.yml
@@ -831,13 +831,16 @@ en:
     past_migrations: Past migrations
     proceed_with_move: Move followers
     redirecting_to: Your account is redirecting to %{acct}.
+    set_redirect: Set redirect
     warning:
       backreference_required: The new account must first be configured to back-reference this one
       before: 'Before proceeding, please read these notes carefully:'
       cooldown: After moving there is a cooldown period during which you will not be able to move again
       disabled_account: Your current account will not be fully usable afterwards. However, you will have access to data export as well as re-activation.
       followers: This action will move all followers from the current account to the new account
+      only_redirect_html: Alternatively, you can <a href="%{path}">only put up a redirect on your profile</a>.
       other_data: No other data will be moved automatically
+      redirect: Your current account's profile will be updated with a redirect notice and be excluded from searches
   moderation:
     title: Moderation
   notification_mailer:
diff --git a/config/routes.rb b/config/routes.rb
index 37e0cbdee..f1a69cf5c 100644
--- a/config/routes.rb
+++ b/config/routes.rb
@@ -134,11 +134,10 @@ Rails.application.routes.draw do
     end
 
     resource :delete, only: [:show, :destroy]
+    resource :migration, only: [:show, :create]
 
-    resource :migration, only: [:show, :create] do
-      collection do
-        post :cancel
-      end
+    namespace :migration do
+      resource :redirect, only: [:new, :create, :destroy]
     end
 
     resources :aliases, only: [:index, :create, :destroy]
-- 
cgit 


From 3babf8464b0903b854ec16d355909444ef3ca0bc Mon Sep 17 00:00:00 2001
From: ThibG <thib@sitedethib.com>
Date: Sun, 29 Sep 2019 22:58:01 +0200
Subject: Add voters count support (#11917)
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

* Add voters count to polls

* Add ActivityPub serialization and parsing of voters count

* Add support for voters count in WebUI

* Move incrementation of voters count out of redis lock

* Reword “voters” to “people”
---
 app/javascript/mastodon/components/poll.js         | 19 +++++++---
 app/lib/activitypub/activity/create.rb             | 40 +++++++++++++++++++---
 app/lib/activitypub/adapter.rb                     |  1 +
 app/models/poll.rb                                 |  1 +
 app/serializers/activitypub/note_serializer.rb     | 12 ++++++-
 app/serializers/rest/poll_serializer.rb            |  2 +-
 app/services/activitypub/process_poll_service.rb   |  5 ++-
 app/services/post_status_service.rb                |  2 +-
 app/services/vote_service.rb                       | 32 +++++++++++++++--
 app/views/statuses/_poll.html.haml                 |  8 +++--
 config/locales/en.yml                              |  3 ++
 .../20190927232842_add_voters_count_to_polls.rb    |  5 +++
 db/schema.rb                                       |  3 +-
 13 files changed, 113 insertions(+), 20 deletions(-)
 create mode 100644 db/migrate/20190927232842_add_voters_count_to_polls.rb

(limited to 'app/models')

diff --git a/app/javascript/mastodon/components/poll.js b/app/javascript/mastodon/components/poll.js
index f88d260f2..cdbcf8f70 100644
--- a/app/javascript/mastodon/components/poll.js
+++ b/app/javascript/mastodon/components/poll.js
@@ -102,10 +102,11 @@ class Poll extends ImmutablePureComponent {
 
   renderOption (option, optionIndex, showResults) {
     const { poll, disabled, intl } = this.props;
-    const percent = poll.get('votes_count') === 0 ? 0 : (option.get('votes_count') / poll.get('votes_count')) * 100;
-    const leading = poll.get('options').filterNot(other => other.get('title') === option.get('title')).every(other => option.get('votes_count') >= other.get('votes_count'));
-    const active  = !!this.state.selected[`${optionIndex}`];
-    const voted   = option.get('voted') || (poll.get('own_votes') && poll.get('own_votes').includes(optionIndex));
+    const pollVotesCount  = poll.get('voters_count') || poll.get('votes_count');
+    const percent         = pollVotesCount === 0 ? 0 : (option.get('votes_count') / pollVotesCount) * 100;
+    const leading         = poll.get('options').filterNot(other => other.get('title') === option.get('title')).every(other => option.get('votes_count') >= other.get('votes_count'));
+    const active          = !!this.state.selected[`${optionIndex}`];
+    const voted           = option.get('voted') || (poll.get('own_votes') && poll.get('own_votes').includes(optionIndex));
 
     let titleEmojified = option.get('title_emojified');
     if (!titleEmojified) {
@@ -157,6 +158,14 @@ class Poll extends ImmutablePureComponent {
     const showResults   = poll.get('voted') || expired;
     const disabled      = this.props.disabled || Object.entries(this.state.selected).every(item => !item);
 
+    let votesCount = null;
+
+    if (poll.get('voters_count') !== null && poll.get('voters_count') !== undefined) {
+      votesCount = <FormattedMessage id='poll.total_people' defaultMessage='{count, plural, one {# person} other {# people}}' values={{ count: poll.get('voters_count') }} />;
+    } else {
+      votesCount = <FormattedMessage id='poll.total_votes' defaultMessage='{count, plural, one {# vote} other {# votes}}' values={{ count: poll.get('votes_count') }} />;
+    }
+
     return (
       <div className='poll'>
         <ul>
@@ -166,7 +175,7 @@ class Poll extends ImmutablePureComponent {
         <div className='poll__footer'>
           {!showResults && <button className='button button-secondary' disabled={disabled} onClick={this.handleVote}><FormattedMessage id='poll.vote' defaultMessage='Vote' /></button>}
           {showResults && !this.props.disabled && <span><button className='poll__link' onClick={this.handleRefresh}><FormattedMessage id='poll.refresh' defaultMessage='Refresh' /></button> · </span>}
-          <FormattedMessage id='poll.total_votes' defaultMessage='{count, plural, one {# vote} other {# votes}}' values={{ count: poll.get('votes_count') }} />
+          {votesCount}
           {poll.get('expires_at') && <span> · {timeRemaining}</span>}
         </div>
       </div>
diff --git a/app/lib/activitypub/activity/create.rb b/app/lib/activitypub/activity/create.rb
index e69193b71..76bf9b2e5 100644
--- a/app/lib/activitypub/activity/create.rb
+++ b/app/lib/activitypub/activity/create.rb
@@ -232,25 +232,40 @@ class ActivityPub::Activity::Create < ActivityPub::Activity
       items    = @object['oneOf']
     end
 
+    voters_count = @object['votersCount']
+
     @account.polls.new(
       multiple: multiple,
       expires_at: expires_at,
       options: items.map { |item| item['name'].presence || item['content'] }.compact,
-      cached_tallies: items.map { |item| item.dig('replies', 'totalItems') || 0 }
+      cached_tallies: items.map { |item| item.dig('replies', 'totalItems') || 0 },
+      voters_count: voters_count
     )
   end
 
   def poll_vote?
     return false if replied_to_status.nil? || replied_to_status.preloadable_poll.nil? || !replied_to_status.local? || !replied_to_status.preloadable_poll.options.include?(@object['name'])
 
-    unless replied_to_status.preloadable_poll.expired?
-      replied_to_status.preloadable_poll.votes.create!(account: @account, choice: replied_to_status.preloadable_poll.options.index(@object['name']), uri: @object['id'])
-      ActivityPub::DistributePollUpdateWorker.perform_in(3.minutes, replied_to_status.id) unless replied_to_status.preloadable_poll.hide_totals?
-    end
+    poll_vote! unless replied_to_status.preloadable_poll.expired?
 
     true
   end
 
+  def poll_vote!
+    poll = replied_to_status.preloadable_poll
+    already_voted = true
+    RedisLock.acquire(poll_lock_options) do |lock|
+      if lock.acquired?
+        already_voted = poll.votes.where(account: @account).exists?
+        poll.votes.create!(account: @account, choice: poll.options.index(@object['name']), uri: @object['id'])
+      else
+        raise Mastodon::RaceConditionError
+      end
+    end
+    increment_voters_count! unless already_voted
+    ActivityPub::DistributePollUpdateWorker.perform_in(3.minutes, replied_to_status.id) unless replied_to_status.preloadable_poll.hide_totals?
+  end
+
   def resolve_thread(status)
     return unless status.reply? && status.thread.nil? && Request.valid_url?(in_reply_to_uri)
     ThreadResolveWorker.perform_async(status.id, in_reply_to_uri)
@@ -416,7 +431,22 @@ class ActivityPub::Activity::Create < ActivityPub::Activity
     ActivityPub::RawDistributionWorker.perform_async(Oj.dump(@json), replied_to_status.account_id, [@account.preferred_inbox_url])
   end
 
+  def increment_voters_count!
+    poll = replied_to_status.preloadable_poll
+    unless poll.voters_count.nil?
+      poll.voters_count = poll.voters_count + 1
+      poll.save
+    end
+  rescue ActiveRecord::StaleObjectError
+    poll.reload
+    retry
+  end
+
   def lock_options
     { redis: Redis.current, key: "create:#{@object['id']}" }
   end
+
+  def poll_lock_options
+    { redis: Redis.current, key: "vote:#{replied_to_status.poll_id}:#{@account.id}" }
+  end
 end
diff --git a/app/lib/activitypub/adapter.rb b/app/lib/activitypub/adapter.rb
index cb2ac72d4..2a8f72333 100644
--- a/app/lib/activitypub/adapter.rb
+++ b/app/lib/activitypub/adapter.rb
@@ -21,6 +21,7 @@ class ActivityPub::Adapter < ActiveModelSerializers::Adapter::Base
     identity_proof: { 'toot' => 'http://joinmastodon.org/ns#', 'IdentityProof' => 'toot:IdentityProof' },
     blurhash: { 'toot' => 'http://joinmastodon.org/ns#', 'blurhash' => 'toot:blurhash' },
     discoverable: { 'toot' => 'http://joinmastodon.org/ns#', 'discoverable' => 'toot:discoverable' },
+    voters_count: { 'toot' => 'http://joinmastodon.org/ns#', 'votersCount' => 'toot:votersCount' },
   }.freeze
 
   def self.default_key_transform
diff --git a/app/models/poll.rb b/app/models/poll.rb
index 55a8f13a6..5427368fd 100644
--- a/app/models/poll.rb
+++ b/app/models/poll.rb
@@ -16,6 +16,7 @@
 #  created_at      :datetime         not null
 #  updated_at      :datetime         not null
 #  lock_version    :integer          default(0), not null
+#  voters_count    :bigint(8)
 #
 
 class Poll < ApplicationRecord
diff --git a/app/serializers/activitypub/note_serializer.rb b/app/serializers/activitypub/note_serializer.rb
index 364d3eda5..110621a28 100644
--- a/app/serializers/activitypub/note_serializer.rb
+++ b/app/serializers/activitypub/note_serializer.rb
@@ -1,7 +1,7 @@
 # frozen_string_literal: true
 
 class ActivityPub::NoteSerializer < ActivityPub::Serializer
-  context_extensions :atom_uri, :conversation, :sensitive
+  context_extensions :atom_uri, :conversation, :sensitive, :voters_count
 
   attributes :id, :type, :summary,
              :in_reply_to, :published, :url,
@@ -23,6 +23,8 @@ class ActivityPub::NoteSerializer < ActivityPub::Serializer
   attribute :end_time, if: :poll_and_expires?
   attribute :closed, if: :poll_and_expired?
 
+  attribute :voters_count, if: :poll_and_voters_count?
+
   def id
     ActivityPub::TagManager.instance.uri_for(object)
   end
@@ -141,6 +143,10 @@ class ActivityPub::NoteSerializer < ActivityPub::Serializer
 
   alias end_time closed
 
+  def voters_count
+    object.preloadable_poll.voters_count
+  end
+
   def poll_and_expires?
     object.preloadable_poll&.expires_at&.present?
   end
@@ -149,6 +155,10 @@ class ActivityPub::NoteSerializer < ActivityPub::Serializer
     object.preloadable_poll&.expired?
   end
 
+  def poll_and_voters_count?
+    object.preloadable_poll&.voters_count
+  end
+
   class MediaAttachmentSerializer < ActivityPub::Serializer
     context_extensions :blurhash, :focal_point
 
diff --git a/app/serializers/rest/poll_serializer.rb b/app/serializers/rest/poll_serializer.rb
index eb98bb2d2..df6ebd0d4 100644
--- a/app/serializers/rest/poll_serializer.rb
+++ b/app/serializers/rest/poll_serializer.rb
@@ -2,7 +2,7 @@
 
 class REST::PollSerializer < ActiveModel::Serializer
   attributes :id, :expires_at, :expired,
-             :multiple, :votes_count
+             :multiple, :votes_count, :voters_count
 
   has_many :loaded_options, key: :options
   has_many :emojis, serializer: REST::CustomEmojiSerializer
diff --git a/app/services/activitypub/process_poll_service.rb b/app/services/activitypub/process_poll_service.rb
index 2fbce65b9..cb4a0d460 100644
--- a/app/services/activitypub/process_poll_service.rb
+++ b/app/services/activitypub/process_poll_service.rb
@@ -28,6 +28,8 @@ class ActivityPub::ProcessPollService < BaseService
       end
     end
 
+    voters_count = @json['votersCount']
+
     latest_options = items.map { |item| item['name'].presence || item['content'] }
 
     # If for some reasons the options were changed, it invalidates all previous
@@ -39,7 +41,8 @@ class ActivityPub::ProcessPollService < BaseService
         last_fetched_at: Time.now.utc,
         expires_at: expires_at,
         options: latest_options,
-        cached_tallies: items.map { |item| item.dig('replies', 'totalItems') || 0 }
+        cached_tallies: items.map { |item| item.dig('replies', 'totalItems') || 0 },
+        voters_count: voters_count
       )
     rescue ActiveRecord::StaleObjectError
       poll.reload
diff --git a/app/services/post_status_service.rb b/app/services/post_status_service.rb
index 34ec6d504..a0a650d62 100644
--- a/app/services/post_status_service.rb
+++ b/app/services/post_status_service.rb
@@ -174,7 +174,7 @@ class PostStatusService < BaseService
   def poll_attributes
     return if @options[:poll].blank?
 
-    @options[:poll].merge(account: @account)
+    @options[:poll].merge(account: @account, voters_count: 0)
   end
 
   def scheduled_options
diff --git a/app/services/vote_service.rb b/app/services/vote_service.rb
index 0eeb8fd56..cb7dce6e8 100644
--- a/app/services/vote_service.rb
+++ b/app/services/vote_service.rb
@@ -12,12 +12,24 @@ class VoteService < BaseService
     @choices = choices
     @votes   = []
 
-    ApplicationRecord.transaction do
-      @choices.each do |choice|
-        @votes << @poll.votes.create!(account: @account, choice: choice)
+    already_voted = true
+
+    RedisLock.acquire(lock_options) do |lock|
+      if lock.acquired?
+        already_voted = @poll.votes.where(account: @account).exists?
+
+        ApplicationRecord.transaction do
+          @choices.each do |choice|
+            @votes << @poll.votes.create!(account: @account, choice: choice)
+          end
+        end
+      else
+        raise Mastodon::RaceConditionError
       end
     end
 
+    increment_voters_count! unless already_voted
+
     ActivityTracker.increment('activity:interactions')
 
     if @poll.account.local?
@@ -53,4 +65,18 @@ class VoteService < BaseService
   def build_json(vote)
     Oj.dump(serialize_payload(vote, ActivityPub::VoteSerializer))
   end
+
+  def increment_voters_count!
+    unless @poll.voters_count.nil?
+      @poll.voters_count = @poll.voters_count + 1
+      @poll.save
+    end
+  rescue ActiveRecord::StaleObjectError
+    @poll.reload
+    retry
+  end
+
+  def lock_options
+    { redis: Redis.current, key: "vote:#{@poll.id}:#{@account.id}" }
+  end
 end
diff --git a/app/views/statuses/_poll.html.haml b/app/views/statuses/_poll.html.haml
index d6b36a5d1..d1aba6ef9 100644
--- a/app/views/statuses/_poll.html.haml
+++ b/app/views/statuses/_poll.html.haml
@@ -1,12 +1,13 @@
 - show_results = (user_signed_in? && poll.voted?(current_account)) || poll.expired?
 - own_votes = user_signed_in? ? poll.own_votes(current_account) : []
+- total_votes_count = poll.voters_count || poll.votes_count
 
 .poll
   %ul
     - poll.loaded_options.each_with_index do |option, index|
       %li
         - if show_results
-          - percent = poll.votes_count > 0 ? 100 * option.votes_count / poll.votes_count : 0
+          - percent = total_votes_count > 0 ? 100 * option.votes_count / total_votes_count : 0
           %span.poll__chart{ style: "width: #{percent}%" }
 
           %label.poll__text><
@@ -24,7 +25,10 @@
       %button.button.button-secondary{ disabled: true }
         = t('statuses.poll.vote')
 
-    %span= t('statuses.poll.total_votes', count: poll.votes_count)
+    - if poll.voters_count.nil?
+      %span= t('statuses.poll.total_votes', count: poll.votes_count)
+    - else
+      %span= t('statuses.poll.total_people', count: poll.voters_count)
 
     - unless poll.expires_at.nil?
       ·
diff --git a/config/locales/en.yml b/config/locales/en.yml
index dbdfe0ca0..82e20cb1f 100644
--- a/config/locales/en.yml
+++ b/config/locales/en.yml
@@ -1030,6 +1030,9 @@ en:
       private: Non-public toot cannot be pinned
       reblog: A boost cannot be pinned
     poll:
+      total_people:
+        one: "%{count} person"
+        other: "%{count} people"
       total_votes:
         one: "%{count} vote"
         other: "%{count} votes"
diff --git a/db/migrate/20190927232842_add_voters_count_to_polls.rb b/db/migrate/20190927232842_add_voters_count_to_polls.rb
new file mode 100644
index 000000000..846385700
--- /dev/null
+++ b/db/migrate/20190927232842_add_voters_count_to_polls.rb
@@ -0,0 +1,5 @@
+class AddVotersCountToPolls < ActiveRecord::Migration[5.2]
+  def change
+    add_column :polls, :voters_count, :bigint
+  end
+end
diff --git a/db/schema.rb b/db/schema.rb
index 8eeaf48a0..557b777e0 100644
--- a/db/schema.rb
+++ b/db/schema.rb
@@ -10,7 +10,7 @@
 #
 # It's strongly recommended that you check this file into your version control system.
 
-ActiveRecord::Schema.define(version: 2019_09_27_124642) do
+ActiveRecord::Schema.define(version: 2019_09_27_232842) do
 
   # These are extensions that must be enabled in order to support this database
   enable_extension "plpgsql"
@@ -529,6 +529,7 @@ ActiveRecord::Schema.define(version: 2019_09_27_124642) do
     t.datetime "created_at", null: false
     t.datetime "updated_at", null: false
     t.integer "lock_version", default: 0, null: false
+    t.bigint "voters_count"
     t.index ["account_id"], name: "index_polls_on_account_id"
     t.index ["status_id"], name: "index_polls_on_status_id"
   end
-- 
cgit 


From 5c42f47617d311219d06e082e4daa41e671903c8 Mon Sep 17 00:00:00 2001
From: Eugen Rochko <eugen@zeonfederated.com>
Date: Tue, 1 Oct 2019 01:19:11 +0200
Subject: Fix records not being indexed sometimes (#12024)

It's possible that after commit callbacks were not firing when
exceptions occurred in the process. Also, the default Sidekiq
strategy does not push indexing jobs immediately, which is not
necessary and could be part of the issue too.
---
 app/models/account.rb                |  2 +-
 app/models/account_stat.rb           |  2 +-
 app/models/application_record.rb     |  6 ++++++
 app/models/favourite.rb              |  2 +-
 app/models/status.rb                 |  2 +-
 app/models/tag.rb                    |  2 +-
 config/application.rb                |  1 +
 config/initializers/chewy.rb         |  5 +++--
 lib/chewy/strategy/custom_sidekiq.rb | 30 ++++++++++++++++++++++++++++++
 spec/rails_helper.rb                 |  2 +-
 10 files changed, 46 insertions(+), 8 deletions(-)
 create mode 100644 lib/chewy/strategy/custom_sidekiq.rb

(limited to 'app/models')

diff --git a/app/models/account.rb b/app/models/account.rb
index 55fe53fae..01d45e36c 100644
--- a/app/models/account.rb
+++ b/app/models/account.rb
@@ -129,7 +129,7 @@ class Account < ApplicationRecord
 
   delegate :chosen_languages, to: :user, prefix: false, allow_nil: true
 
-  update_index('accounts#account', :self) if Chewy.enabled?
+  update_index('accounts#account', :self)
 
   def local?
     domain.nil?
diff --git a/app/models/account_stat.rb b/app/models/account_stat.rb
index 6d1097cec..1351f7d8a 100644
--- a/app/models/account_stat.rb
+++ b/app/models/account_stat.rb
@@ -16,7 +16,7 @@
 class AccountStat < ApplicationRecord
   belongs_to :account, inverse_of: :account_stat
 
-  update_index('accounts#account', :account) if Chewy.enabled?
+  update_index('accounts#account', :account)
 
   def increment_count!(key)
     update(attributes_for_increment(key))
diff --git a/app/models/application_record.rb b/app/models/application_record.rb
index c1b873da6..5d7d3a096 100644
--- a/app/models/application_record.rb
+++ b/app/models/application_record.rb
@@ -5,6 +5,12 @@ class ApplicationRecord < ActiveRecord::Base
 
   include Remotable
 
+  class << self
+    def update_index(_type_name, *_args, &_block)
+      super if Chewy.enabled?
+    end
+  end
+
   def boolean_with_default(key, default_value)
     value = attributes[key]
 
diff --git a/app/models/favourite.rb b/app/models/favourite.rb
index 17f8c9fa6..bf0ec4449 100644
--- a/app/models/favourite.rb
+++ b/app/models/favourite.rb
@@ -13,7 +13,7 @@
 class Favourite < ApplicationRecord
   include Paginable
 
-  update_index('statuses#status', :status) if Chewy.enabled?
+  update_index('statuses#status', :status)
 
   belongs_to :account, inverse_of: :favourites
   belongs_to :status,  inverse_of: :favourites
diff --git a/app/models/status.rb b/app/models/status.rb
index 5e7474577..078a64566 100644
--- a/app/models/status.rb
+++ b/app/models/status.rb
@@ -39,7 +39,7 @@ class Status < ApplicationRecord
   # will be based on current time instead of `created_at`
   attr_accessor :override_timestamps
 
-  update_index('statuses#status', :proper) if Chewy.enabled?
+  update_index('statuses#status', :proper)
 
   enum visibility: [:public, :unlisted, :private, :direct, :limited], _suffix: :visibility
 
diff --git a/app/models/tag.rb b/app/models/tag.rb
index 9aca3983f..82786daa8 100644
--- a/app/models/tag.rb
+++ b/app/models/tag.rb
@@ -49,7 +49,7 @@ class Tag < ApplicationRecord
 
   after_save :save_account_tag_stat
 
-  update_index('tags#tag', :self) if Chewy.enabled?
+  update_index('tags#tag', :self)
 
   def account_tag_stat
     super || build_account_tag_stat
diff --git a/config/application.rb b/config/application.rb
index 3ced81b8f..60f73f8bb 100644
--- a/config/application.rb
+++ b/config/application.rb
@@ -15,6 +15,7 @@ require_relative '../lib/mastodon/snowflake'
 require_relative '../lib/mastodon/version'
 require_relative '../lib/devise/two_factor_ldap_authenticatable'
 require_relative '../lib/devise/two_factor_pam_authenticatable'
+require_relative '../lib/chewy/strategy/custom_sidekiq'
 
 Dotenv::Railtie.load
 
diff --git a/config/initializers/chewy.rb b/config/initializers/chewy.rb
index d5347f2bf..9ff0dccc1 100644
--- a/config/initializers/chewy.rb
+++ b/config/initializers/chewy.rb
@@ -12,8 +12,9 @@ Chewy.settings = {
   sidekiq: { queue: 'pull' },
 }
 
-Chewy.root_strategy    = enabled ? :sidekiq : :bypass
-Chewy.request_strategy = enabled ? :sidekiq : :bypass
+Chewy.root_strategy              = :custom_sidekiq
+Chewy.request_strategy           = :custom_sidekiq
+Chewy.use_after_commit_callbacks = false
 
 module Chewy
   class << self
diff --git a/lib/chewy/strategy/custom_sidekiq.rb b/lib/chewy/strategy/custom_sidekiq.rb
new file mode 100644
index 000000000..3e54326ba
--- /dev/null
+++ b/lib/chewy/strategy/custom_sidekiq.rb
@@ -0,0 +1,30 @@
+# frozen_string_literal: true
+
+module Chewy
+  class Strategy
+    class CustomSidekiq < Base
+      class Worker
+        include ::Sidekiq::Worker
+
+        sidekiq_options queue: 'pull'
+
+        def perform(type, ids, options = {})
+          options[:refresh] = !Chewy.disable_refresh_async if Chewy.disable_refresh_async
+          type.constantize.import!(ids, options)
+        end
+      end
+
+      def update(type, objects, _options = {})
+        return unless Chewy.enabled?
+
+        ids = type.root.id ? Array.wrap(objects) : type.adapter.identify(objects)
+
+        return if ids.empty?
+
+        Worker.perform_async(type.name, ids)
+      end
+
+      def leave; end
+    end
+  end
+end
diff --git a/spec/rails_helper.rb b/spec/rails_helper.rb
index 3a5e7491e..6fbceca53 100644
--- a/spec/rails_helper.rb
+++ b/spec/rails_helper.rb
@@ -12,7 +12,7 @@ require 'capybara/rspec'
 Dir[Rails.root.join('spec/support/**/*.rb')].each { |f| require f }
 
 ActiveRecord::Migration.maintain_test_schema!
-WebMock.disable_net_connect!
+WebMock.disable_net_connect!(allow: Chewy.settings[:host])
 Redis.current = Redis::Namespace.new("mastodon_test#{ENV['TEST_ENV_NUMBER']}", redis: Redis.current)
 Sidekiq::Testing.inline!
 Sidekiq::Logging.logger = nil
-- 
cgit 


From 62f60e86c281ae1cd0356a7630dbb76069e2ffde Mon Sep 17 00:00:00 2001
From: Eugen Rochko <eugen@zeonfederated.com>
Date: Wed, 2 Oct 2019 04:59:37 +0200
Subject: Fix account counters being overwritten by parallel writes (#12045)

---
 app/models/account_stat.rb                         | 22 +++++++++
 ...1001213028_add_lock_version_to_account_stats.rb | 15 ++++++
 db/schema.rb                                       |  3 +-
 spec/models/account_stat_spec.rb                   | 53 ++++++++++++++++++++++
 4 files changed, 92 insertions(+), 1 deletion(-)
 create mode 100644 db/migrate/20191001213028_add_lock_version_to_account_stats.rb

(limited to 'app/models')

diff --git a/app/models/account_stat.rb b/app/models/account_stat.rb
index 1351f7d8a..c84e4217c 100644
--- a/app/models/account_stat.rb
+++ b/app/models/account_stat.rb
@@ -11,6 +11,7 @@
 #  created_at      :datetime         not null
 #  updated_at      :datetime         not null
 #  last_status_at  :datetime
+#  lock_version    :integer          default(0), not null
 #
 
 class AccountStat < ApplicationRecord
@@ -20,10 +21,26 @@ class AccountStat < ApplicationRecord
 
   def increment_count!(key)
     update(attributes_for_increment(key))
+  rescue ActiveRecord::StaleObjectError
+    begin
+      reload_with_id
+    rescue ActiveRecord::RecordNotFound
+      # Nothing to do
+    else
+      retry
+    end
   end
 
   def decrement_count!(key)
     update(key => [public_send(key) - 1, 0].max)
+  rescue ActiveRecord::StaleObjectError
+    begin
+      reload_with_id
+    rescue ActiveRecord::RecordNotFound
+      # Nothing to do
+    else
+      retry
+    end
   end
 
   private
@@ -33,4 +50,9 @@ class AccountStat < ApplicationRecord
     attrs[:last_status_at] = Time.now.utc if key == :statuses_count
     attrs
   end
+
+  def reload_with_id
+    self.id = find_by!(account: account).id if new_record?
+    reload
+  end
 end
diff --git a/db/migrate/20191001213028_add_lock_version_to_account_stats.rb b/db/migrate/20191001213028_add_lock_version_to_account_stats.rb
new file mode 100644
index 000000000..47f37cca2
--- /dev/null
+++ b/db/migrate/20191001213028_add_lock_version_to_account_stats.rb
@@ -0,0 +1,15 @@
+require Rails.root.join('lib', 'mastodon', 'migration_helpers')
+
+class AddLockVersionToAccountStats < ActiveRecord::Migration[5.2]
+  include Mastodon::MigrationHelpers
+
+  disable_ddl_transaction!
+
+  def up
+    safety_assured { add_column_with_default :account_stats, :lock_version, :integer, allow_null: false, default: 0 }
+  end
+
+  def down
+    remove_column :account_stats, :lock_version
+  end
+end
diff --git a/db/schema.rb b/db/schema.rb
index 557b777e0..2d516965c 100644
--- a/db/schema.rb
+++ b/db/schema.rb
@@ -10,7 +10,7 @@
 #
 # It's strongly recommended that you check this file into your version control system.
 
-ActiveRecord::Schema.define(version: 2019_09_27_232842) do
+ActiveRecord::Schema.define(version: 2019_10_01_213028) do
 
   # These are extensions that must be enabled in order to support this database
   enable_extension "plpgsql"
@@ -97,6 +97,7 @@ ActiveRecord::Schema.define(version: 2019_09_27_232842) do
     t.datetime "created_at", null: false
     t.datetime "updated_at", null: false
     t.datetime "last_status_at"
+    t.integer "lock_version", default: 0, null: false
     t.index ["account_id"], name: "index_account_stats_on_account_id", unique: true
   end
 
diff --git a/spec/models/account_stat_spec.rb b/spec/models/account_stat_spec.rb
index a94185109..8adc0d1d6 100644
--- a/spec/models/account_stat_spec.rb
+++ b/spec/models/account_stat_spec.rb
@@ -1,4 +1,57 @@
 require 'rails_helper'
 
 RSpec.describe AccountStat, type: :model do
+  describe '#increment_count!' do
+    it 'increments the count' do
+      account_stat = AccountStat.create(account: Fabricate(:account))
+      expect(account_stat.followers_count).to eq 0
+      account_stat.increment_count!(:followers_count)
+      expect(account_stat.followers_count).to eq 1
+    end
+
+    it 'increments the count in multi-threaded an environment' do
+      account_stat   = AccountStat.create(account: Fabricate(:account), statuses_count: 0)
+      increment_by   = 15
+      wait_for_start = true
+
+      threads = Array.new(increment_by) do
+        Thread.new do
+          true while wait_for_start
+          AccountStat.find(account_stat.id).increment_count!(:statuses_count)
+        end
+      end
+
+      wait_for_start = false
+      threads.each(&:join)
+
+      expect(account_stat.reload.statuses_count).to eq increment_by
+    end
+  end
+
+  describe '#decrement_count!' do
+    it 'decrements the count' do
+      account_stat = AccountStat.create(account: Fabricate(:account), followers_count: 15)
+      expect(account_stat.followers_count).to eq 15
+      account_stat.decrement_count!(:followers_count)
+      expect(account_stat.followers_count).to eq 14
+    end
+
+    it 'decrements the count in multi-threaded an environment' do
+      account_stat   = AccountStat.create(account: Fabricate(:account), statuses_count: 15)
+      decrement_by   = 10
+      wait_for_start = true
+
+      threads = Array.new(decrement_by) do
+        Thread.new do
+          true while wait_for_start
+          AccountStat.find(account_stat.id).decrement_count!(:statuses_count)
+        end
+      end
+
+      wait_for_start = false
+      threads.each(&:join)
+
+      expect(account_stat.reload.statuses_count).to eq 5
+    end
+  end
 end
-- 
cgit 


From 575dc11cb2045c32b8fc0325de7bc321bd4728aa Mon Sep 17 00:00:00 2001
From: Eugen Rochko <eugen@zeonfederated.com>
Date: Wed, 2 Oct 2019 20:04:46 +0200
Subject: Fix needlessly indexing unsearchable statuses into ElasticSearch
 (#12041)

---
 app/chewy/statuses_index.rb |  8 ++++----
 app/models/status.rb        | 10 ++++++----
 2 files changed, 10 insertions(+), 8 deletions(-)

(limited to 'app/models')

diff --git a/app/chewy/statuses_index.rb b/app/chewy/statuses_index.rb
index b7365d5ca..f5735421c 100644
--- a/app/chewy/statuses_index.rb
+++ b/app/chewy/statuses_index.rb
@@ -31,19 +31,19 @@ class StatusesIndex < Chewy::Index
     },
   }
 
-  define_type ::Status.unscoped.without_reblogs.includes(:media_attachments) do
+  define_type ::Status.unscoped.kept.without_reblogs.includes(:media_attachments), delete_if: ->(status) { status.searchable_by.empty? } do
     crutch :mentions do |collection|
-      data = ::Mention.where(status_id: collection.map(&:id)).pluck(:status_id, :account_id)
+      data = ::Mention.where(status_id: collection.map(&:id)).where(account: Account.local).pluck(:status_id, :account_id)
       data.each.with_object({}) { |(id, name), result| (result[id] ||= []).push(name) }
     end
 
     crutch :favourites do |collection|
-      data = ::Favourite.where(status_id: collection.map(&:id)).pluck(:status_id, :account_id)
+      data = ::Favourite.where(status_id: collection.map(&:id)).where(account: Account.local).pluck(:status_id, :account_id)
       data.each.with_object({}) { |(id, name), result| (result[id] ||= []).push(name) }
     end
 
     crutch :reblogs do |collection|
-      data = ::Status.where(reblog_of_id: collection.map(&:id)).pluck(:reblog_of_id, :account_id)
+      data = ::Status.where(reblog_of_id: collection.map(&:id)).where(account: Account.local).pluck(:reblog_of_id, :account_id)
       data.each.with_object({}) { |(id, name), result| (result[id] ||= []).push(name) }
     end
 
diff --git a/app/models/status.rb b/app/models/status.rb
index 078a64566..3504ac370 100644
--- a/app/models/status.rb
+++ b/app/models/status.rb
@@ -129,12 +129,14 @@ class Status < ApplicationRecord
   REAL_TIME_WINDOW = 6.hours
 
   def searchable_by(preloaded = nil)
-    ids = [account_id]
+    ids = []
+
+    ids << account_id if local?
 
     if preloaded.nil?
-      ids += mentions.pluck(:account_id)
-      ids += favourites.pluck(:account_id)
-      ids += reblogs.pluck(:account_id)
+      ids += mentions.where(account: Account.local).pluck(:account_id)
+      ids += favourites.where(account: Account.local).pluck(:account_id)
+      ids += reblogs.where(account: Account.local).pluck(:account_id)
     else
       ids += preloaded.mentions[id] || []
       ids += preloaded.favourites[id] || []
-- 
cgit 


From ca22a22d7f4db867ad0045a7978e3d8dcd251a69 Mon Sep 17 00:00:00 2001
From: Eugen Rochko <eugen@zeonfederated.com>
Date: Thu, 3 Oct 2019 01:09:12 +0200
Subject: Fix performance of GIF re-encoding (#12057)

* Change animated GIF detection to not shell out to ImageMagick

Signed-off-by: Eugen Rochko <eugen@zeonfederated.com>

* Change video encoding parameters to limit to 10800 video frames

Signed-off-by: Eugen Rochko <eugen@zeonfederated.com>

* Limit GIF image size further

Signed-off-by: Eugen Rochko <eugen@zeonfederated.com>

* Always strip metadata from video files

* Fix code style issues
---
 app/models/concerns/attachmentable.rb |   4 +-
 app/models/media_attachment.rb        |  33 ++++++++---
 lib/paperclip/gif_transcoder.rb       | 101 +++++++++++++++++++++++++++++++++-
 lib/paperclip/video_transcoder.rb     |   2 +
 4 files changed, 128 insertions(+), 12 deletions(-)

(limited to 'app/models')

diff --git a/app/models/concerns/attachmentable.rb b/app/models/concerns/attachmentable.rb
index 246c2c27c..3bbc6453c 100644
--- a/app/models/concerns/attachmentable.rb
+++ b/app/models/concerns/attachmentable.rb
@@ -6,6 +6,7 @@ module Attachmentable
   extend ActiveSupport::Concern
 
   MAX_MATRIX_LIMIT = 16_777_216 # 4096x4096px or approx. 16MB
+  GIF_MATRIX_LIMIT = 921_600    # 1280x720px
 
   included do
     before_post_process :set_file_extensions
@@ -42,8 +43,9 @@ module Attachmentable
       next if attachment.blank? || !/image.*/.match?(attachment.content_type) || attachment.queued_for_write[:original].blank?
 
       width, height = FastImage.size(attachment.queued_for_write[:original].path)
+      matrix_limit  = attachment.content_type == 'image/gif' ? GIF_MATRIX_LIMIT : MAX_MATRIX_LIMIT
 
-      raise Mastodon::DimensionsValidationError, "#{width}x#{height} images are not supported, must be below #{MAX_MATRIX_LIMIT} sqpx" if width.present? && height.present? && (width * height >= MAX_MATRIX_LIMIT)
+      raise Mastodon::DimensionsValidationError, "#{width}x#{height} images are not supported" if width.present? && height.present? && (width * height > matrix_limit)
     end
   end
 
diff --git a/app/models/media_attachment.rb b/app/models/media_attachment.rb
index c4932f2ef..630dab55a 100644
--- a/app/models/media_attachment.rb
+++ b/app/models/media_attachment.rb
@@ -65,6 +65,17 @@ class MediaAttachment < ApplicationRecord
       file_geometry_parser: FastGeometryParser,
       blurhash: BLURHASH_OPTIONS,
     },
+
+    original: {
+      keep_same_format: true,
+      convert_options: {
+        output: {
+          'map_metadata' => '-1',
+          'c:v' => 'copy',
+          'c:a' => 'copy',
+        },
+      },
+    },
   }.freeze
 
   AUDIO_STYLES = {
@@ -86,14 +97,15 @@ class MediaAttachment < ApplicationRecord
       output: {
         'loglevel' => 'fatal',
         'movflags' => 'faststart',
-        'pix_fmt'  => 'yuv420p',
-        'vf'       => 'scale=\'trunc(iw/2)*2:trunc(ih/2)*2\'',
-        'vsync'    => 'cfr',
-        'c:v'      => 'h264',
-        'b:v'      => '500K',
-        'maxrate'  => '1300K',
-        'bufsize'  => '1300K',
-        'crf'      => 18,
+        'pix_fmt' => 'yuv420p',
+        'vf' => 'scale=\'trunc(iw/2)*2:trunc(ih/2)*2\'',
+        'vsync' => 'cfr',
+        'c:v' => 'h264',
+        'maxrate' => '1300K',
+        'bufsize' => '1300K',
+        'frames:v' => 60 * 60 * 3,
+        'crf' => 18,
+        'map_metadata' => '-1',
       },
     },
   }.freeze
@@ -103,7 +115,7 @@ class MediaAttachment < ApplicationRecord
     original: VIDEO_FORMAT,
   }.freeze
 
-  IMAGE_LIMIT = 8.megabytes
+  IMAGE_LIMIT = 10.megabytes
   VIDEO_LIMIT = 40.megabytes
 
   belongs_to :account,          inverse_of: :media_attachments, optional: true
@@ -244,7 +256,9 @@ class MediaAttachment < ApplicationRecord
 
   def set_meta
     meta = populate_meta
+
     return if meta == {}
+
     file.instance_write :meta, meta
   end
 
@@ -287,6 +301,7 @@ class MediaAttachment < ApplicationRecord
 
   def reset_parent_cache
     return if status_id.nil?
+
     Rails.cache.delete("statuses/#{status_id}")
   end
 end
diff --git a/lib/paperclip/gif_transcoder.rb b/lib/paperclip/gif_transcoder.rb
index cbab6fd99..64f12f963 100644
--- a/lib/paperclip/gif_transcoder.rb
+++ b/lib/paperclip/gif_transcoder.rb
@@ -1,5 +1,103 @@
 # frozen_string_literal: true
 
+class GifReader
+  attr_reader :animated
+
+  EXTENSION_LABELS = [0xf9, 0x01, 0xff].freeze
+  GIF_HEADERS      = %w(GIF87a GIF89a).freeze
+
+  class GifReaderException; end
+
+  class UnknownImageType < GifReaderException; end
+
+  class CannotParseImage < GifReaderException; end
+
+  def self.animated?(path)
+    new(path).animated
+  rescue GifReaderException
+    false
+  end
+
+  def initialize(path, max_frames = 2)
+    @path      = path
+    @nb_frames = 0
+
+    File.open(path, 'rb') do |s|
+      raise UnknownImageType unless GIF_HEADERS.include?(s.read(6))
+
+      # Skip to "packed byte"
+      s.seek(4, IO::SEEK_CUR)
+
+      # "Packed byte" gives us the size of the GIF color table
+      packed_byte, = s.read(1).unpack('C')
+
+      # Skip background color and aspect ratio
+      s.seek(2, IO::SEEK_CUR)
+
+      if packed_byte & 0x80 != 0
+        # GIF uses a global color table, skip it
+        s.seek(3 * (1 << ((packed_byte & 0x07) + 1)), IO::SEEK_CUR)
+      end
+
+      # Now read data
+      while @nb_frames < max_frames
+        separator = s.read(1)
+
+        case separator
+        when ',' # Image block
+          @nb_frames += 1
+
+          # Skip to "packed byte"
+          s.seek(8, IO::SEEK_CUR)
+          packed_byte, = s.read(1).unpack('C')
+
+          if packed_byte & 0x80 != 0
+            # Image uses a local color table, skip it
+            s.seek(3 * (1 << ((packed_byte & 0x07) + 1)), IO::SEEK_CUR)
+          end
+
+          # Skip lzw min code size
+          raise InvalidValue unless s.read(1).unpack('C')[0] >= 2
+
+          # Skip image data sub-blocks
+          skip_sub_blocks!(s)
+        when '!' # Extension block
+          skip_extension_block!(s)
+        when ';' # Trailer
+          break
+        else
+          raise CannotParseImage
+        end
+      end
+    end
+
+    @animated = @nb_frames > 1
+  end
+
+  private
+
+  def skip_extension_block!(file)
+    if EXTENSION_LABELS.include?(file.read(1).unpack('C')[0])
+      block_size, = file.read(1).unpack('C')
+      file.seek(block_size, IO::SEEK_CUR)
+    end
+
+    # Read until extension block end marker
+    skip_sub_blocks!(file)
+  end
+
+  # Skip sub-blocks up until block end marker
+  def skip_sub_blocks!(file)
+    loop do
+      size, = file.read(1).unpack('C')
+
+      break if size.zero?
+
+      file.seek(size, IO::SEEK_CUR)
+    end
+  end
+end
+
 module Paperclip
   # This transcoder is only to be used for the MediaAttachment model
   # to convert animated gifs to webm
@@ -19,8 +117,7 @@ module Paperclip
     private
 
     def needs_convert?
-      num_frames = identify('-format %n :file', file: file.path).to_i
-      options[:style] == :original && num_frames > 1
+      options[:style] == :original && GifReader.animated?(file.path)
     end
   end
 end
diff --git a/lib/paperclip/video_transcoder.rb b/lib/paperclip/video_transcoder.rb
index c3504c17c..66f7feda5 100644
--- a/lib/paperclip/video_transcoder.rb
+++ b/lib/paperclip/video_transcoder.rb
@@ -6,7 +6,9 @@ module Paperclip
   class VideoTranscoder < Paperclip::Processor
     def make
       meta = ::Av.cli.identify(@file.path)
+
       attachment.instance.type = MediaAttachment.types[:gifv] unless meta[:audio_encode]
+      options[:format] = File.extname(attachment.instance.file_file_name)[1..-1] if options[:keep_same_format]
 
       Paperclip::Transcoder.make(file, options, attachment)
     end
-- 
cgit 


From a7917269d30104d0ba1e310d5f30f72891c9a37b Mon Sep 17 00:00:00 2001
From: Hugo Gameiro <hmgameiro@gmail.com>
Date: Sun, 6 Oct 2019 18:48:26 +0100
Subject: add loglevel fatal to video and audio styles (#12088)

---
 app/models/media_attachment.rb | 3 +++
 1 file changed, 3 insertions(+)

(limited to 'app/models')

diff --git a/app/models/media_attachment.rb b/app/models/media_attachment.rb
index 630dab55a..9c6c04556 100644
--- a/app/models/media_attachment.rb
+++ b/app/models/media_attachment.rb
@@ -57,6 +57,7 @@ class MediaAttachment < ApplicationRecord
     small: {
       convert_options: {
         output: {
+          'loglevel' => 'fatal',
           vf: 'scale=\'min(400\, iw):min(400\, ih)\':force_original_aspect_ratio=decrease',
         },
       },
@@ -70,6 +71,7 @@ class MediaAttachment < ApplicationRecord
       keep_same_format: true,
       convert_options: {
         output: {
+          'loglevel' => 'fatal',
           'map_metadata' => '-1',
           'c:v' => 'copy',
           'c:a' => 'copy',
@@ -84,6 +86,7 @@ class MediaAttachment < ApplicationRecord
       content_type: 'audio/mpeg',
       convert_options: {
         output: {
+          'loglevel' => 'fatal',
           'q:a' => 2,
         },
       },
-- 
cgit 


From f665901e3c0930fb8b3741f6bc6f6a15dd0343f6 Mon Sep 17 00:00:00 2001
From: Eugen Rochko <eugen@zeonfederated.com>
Date: Sun, 6 Oct 2019 22:11:17 +0200
Subject: Fix performance of home feed regeneration (#12084)

Fetching statuses from all followed accounts at once takes too long
within Postgres. Fetching them one by one and merging in Ruby
could be a lot less resource-intensive

Because the query for dynamically fetching the home timeline is so
heavy, we can no longer offer it when the home timeline is missing
---
 .../api/v1/timelines/home_controller.rb            |   6 +-
 app/javascript/mastodon/actions/timelines.js       |   2 +-
 .../mastodon/components/missing_indicator.js       |  23 +++--
 .../mastodon/components/regeneration_indicator.js  |  18 ++++
 app/javascript/mastodon/components/status_list.js  |  15 +--
 .../mastodon/features/generic_not_found/index.js   |   2 +-
 app/javascript/styles/mastodon/components.scss     |  30 ++----
 app/javascript/styles/mastodon/introduction.scss   |   3 +-
 app/lib/feed_manager.rb                            | 106 +++++++++++++++------
 app/models/home_feed.rb                            |  16 +---
 app/models/status.rb                               |   4 -
 spec/models/home_feed_spec.rb                      |   5 +-
 spec/models/status_spec.rb                         |  43 ---------
 13 files changed, 132 insertions(+), 141 deletions(-)
 create mode 100644 app/javascript/mastodon/components/regeneration_indicator.js

(limited to 'app/models')

diff --git a/app/controllers/api/v1/timelines/home_controller.rb b/app/controllers/api/v1/timelines/home_controller.rb
index fcd0757f1..ff5ede138 100644
--- a/app/controllers/api/v1/timelines/home_controller.rb
+++ b/app/controllers/api/v1/timelines/home_controller.rb
@@ -13,7 +13,7 @@ class Api::V1::Timelines::HomeController < Api::BaseController
     render json: @statuses,
            each_serializer: REST::StatusSerializer,
            relationships: StatusRelationshipsPresenter.new(@statuses, current_user&.account_id),
-           status: regeneration_in_progress? ? 206 : 200
+           status: account_home_feed.regenerating? ? 206 : 200
   end
 
   private
@@ -62,8 +62,4 @@ class Api::V1::Timelines::HomeController < Api::BaseController
   def pagination_since_id
     @statuses.first.id
   end
-
-  def regeneration_in_progress?
-    Redis.current.exists("account:#{current_account.id}:regeneration")
-  end
 end
diff --git a/app/javascript/mastodon/actions/timelines.js b/app/javascript/mastodon/actions/timelines.js
index 7eeba2aa7..bc2ac5e82 100644
--- a/app/javascript/mastodon/actions/timelines.js
+++ b/app/javascript/mastodon/actions/timelines.js
@@ -97,7 +97,7 @@ export function expandTimeline(timelineId, path, params = {}, done = noOp) {
     api(getState).get(path, { params }).then(response => {
       const next = getLinks(response).refs.find(link => link.rel === 'next');
       dispatch(importFetchedStatuses(response.data));
-      dispatch(expandTimelineSuccess(timelineId, response.data, next ? next.uri : null, response.code === 206, isLoadingRecent, isLoadingMore, isLoadingRecent && preferPendingItems));
+      dispatch(expandTimelineSuccess(timelineId, response.data, next ? next.uri : null, response.status === 206, isLoadingRecent, isLoadingMore, isLoadingRecent && preferPendingItems));
       done();
     }).catch(error => {
       dispatch(expandTimelineFail(timelineId, error, isLoadingMore));
diff --git a/app/javascript/mastodon/components/missing_indicator.js b/app/javascript/mastodon/components/missing_indicator.js
index 70d8c3b98..7b0101bab 100644
--- a/app/javascript/mastodon/components/missing_indicator.js
+++ b/app/javascript/mastodon/components/missing_indicator.js
@@ -1,17 +1,24 @@
 import React from 'react';
+import PropTypes from 'prop-types';
 import { FormattedMessage } from 'react-intl';
+import illustration from 'mastodon/../images/elephant_ui_disappointed.svg';
+import classNames from 'classnames';
 
-const MissingIndicator = () => (
-  <div className='regeneration-indicator missing-indicator'>
-    <div>
-      <div className='regeneration-indicator__figure' />
+const MissingIndicator = ({ fullPage }) => (
+  <div className={classNames('regeneration-indicator', { 'regeneration-indicator--without-header': fullPage })}>
+    <div className='regeneration-indicator__figure'>
+      <img src={illustration} alt='' />
+    </div>
 
-      <div className='regeneration-indicator__label'>
-        <FormattedMessage id='missing_indicator.label' tagName='strong' defaultMessage='Not found' />
-        <FormattedMessage id='missing_indicator.sublabel' defaultMessage='This resource could not be found' />
-      </div>
+    <div className='regeneration-indicator__label'>
+      <FormattedMessage id='missing_indicator.label' tagName='strong' defaultMessage='Not found' />
+      <FormattedMessage id='missing_indicator.sublabel' defaultMessage='This resource could not be found' />
     </div>
   </div>
 );
 
+MissingIndicator.propTypes = {
+  fullPage: PropTypes.bool,
+};
+
 export default MissingIndicator;
diff --git a/app/javascript/mastodon/components/regeneration_indicator.js b/app/javascript/mastodon/components/regeneration_indicator.js
new file mode 100644
index 000000000..faf88c6b5
--- /dev/null
+++ b/app/javascript/mastodon/components/regeneration_indicator.js
@@ -0,0 +1,18 @@
+import React from 'react';
+import { FormattedMessage } from 'react-intl';
+import illustration from 'mastodon/../images/elephant_ui_working.svg';
+
+const MissingIndicator = () => (
+  <div className='regeneration-indicator'>
+    <div className='regeneration-indicator__figure'>
+      <img src={illustration} alt='' />
+    </div>
+
+    <div className='regeneration-indicator__label'>
+      <FormattedMessage id='regeneration_indicator.label' tagName='strong' defaultMessage='Loading&hellip;' />
+      <FormattedMessage id='regeneration_indicator.sublabel' defaultMessage='Your home feed is being prepared!' />
+    </div>
+  </div>
+);
+
+export default MissingIndicator;
diff --git a/app/javascript/mastodon/components/status_list.js b/app/javascript/mastodon/components/status_list.js
index 745e6422d..e1b370c91 100644
--- a/app/javascript/mastodon/components/status_list.js
+++ b/app/javascript/mastodon/components/status_list.js
@@ -1,12 +1,12 @@
 import { debounce } from 'lodash';
 import React from 'react';
-import { FormattedMessage } from 'react-intl';
 import ImmutablePropTypes from 'react-immutable-proptypes';
 import PropTypes from 'prop-types';
 import StatusContainer from '../containers/status_container';
 import ImmutablePureComponent from 'react-immutable-pure-component';
 import LoadGap from './load_gap';
 import ScrollableList from './scrollable_list';
+import RegenerationIndicator from 'mastodon/components/regeneration_indicator';
 
 export default class StatusList extends ImmutablePureComponent {
 
@@ -81,18 +81,7 @@ export default class StatusList extends ImmutablePureComponent {
     const { isLoading, isPartial } = other;
 
     if (isPartial) {
-      return (
-        <div className='regeneration-indicator'>
-          <div>
-            <div className='regeneration-indicator__figure' />
-
-            <div className='regeneration-indicator__label'>
-              <FormattedMessage id='regeneration_indicator.label' tagName='strong' defaultMessage='Loading&hellip;' />
-              <FormattedMessage id='regeneration_indicator.sublabel' defaultMessage='Your home feed is being prepared!' />
-            </div>
-          </div>
-        </div>
-      );
+      return <RegenerationIndicator />;
     }
 
     let scrollableContent = (isLoading || statusIds.size > 0) ? (
diff --git a/app/javascript/mastodon/features/generic_not_found/index.js b/app/javascript/mastodon/features/generic_not_found/index.js
index 0290be47f..41cd61a5f 100644
--- a/app/javascript/mastodon/features/generic_not_found/index.js
+++ b/app/javascript/mastodon/features/generic_not_found/index.js
@@ -4,7 +4,7 @@ import MissingIndicator from '../../components/missing_indicator';
 
 const GenericNotFound = () => (
   <Column>
-    <MissingIndicator />
+    <MissingIndicator fullPage />
   </Column>
 );
 
diff --git a/app/javascript/styles/mastodon/components.scss b/app/javascript/styles/mastodon/components.scss
index a6675b8ed..d9182ade9 100644
--- a/app/javascript/styles/mastodon/components.scss
+++ b/app/javascript/styles/mastodon/components.scss
@@ -3127,37 +3127,27 @@ a.status-card.compact:hover {
   cursor: default;
   display: flex;
   flex: 1 1 auto;
+  flex-direction: column;
   align-items: center;
   justify-content: center;
   padding: 20px;
 
-  & > div {
-    width: 100%;
-    background: transparent;
-    padding-top: 0;
-  }
-
   &__figure {
-    background: url('../images/elephant_ui_working.svg') no-repeat center 0;
-    width: 100%;
-    height: 160px;
-    background-size: contain;
-    position: absolute;
-    top: 50%;
-    left: 50%;
-    transform: translate(-50%, -50%);
+    &,
+    img {
+      display: block;
+      width: auto;
+      height: 160px;
+      margin: 0;
+    }
   }
 
-  &.missing-indicator {
+  &--without-header {
     padding-top: 20px + 48px;
-
-    .regeneration-indicator__figure {
-      background-image: url('../images/elephant_ui_disappointed.svg');
-    }
   }
 
   &__label {
-    margin-top: 200px;
+    margin-top: 30px;
 
     strong {
       display: block;
diff --git a/app/javascript/styles/mastodon/introduction.scss b/app/javascript/styles/mastodon/introduction.scss
index 222d8f60e..b44ae7306 100644
--- a/app/javascript/styles/mastodon/introduction.scss
+++ b/app/javascript/styles/mastodon/introduction.scss
@@ -3,9 +3,10 @@
   flex-direction: column;
   justify-content: center;
   align-items: center;
+  height: 100vh;
+  background: $ui-base-color;
 
   @media screen and (max-width: 920px) {
-    background: darken($ui-base-color, 8%);
     display: block !important;
   }
 
diff --git a/app/lib/feed_manager.rb b/app/lib/feed_manager.rb
index 871ec5c19..d8b486b60 100644
--- a/app/lib/feed_manager.rb
+++ b/app/lib/feed_manager.rb
@@ -19,7 +19,7 @@ class FeedManager
 
   def filter?(timeline_type, status, receiver_id)
     if timeline_type == :home
-      filter_from_home?(status, receiver_id)
+      filter_from_home?(status, receiver_id, build_crutches(receiver_id, [status]))
     elsif timeline_type == :mentions
       filter_from_mentions?(status, receiver_id)
     else
@@ -29,6 +29,7 @@ class FeedManager
 
   def push_to_home(account, status)
     return false unless add_to_feed(:home, account.id, status, account.user&.aggregates_reblogs?)
+
     trim(:home, account.id)
     PushUpdateWorker.perform_async(account.id, status.id, "timeline:#{account.id}") if push_update_required?("timeline:#{account.id}")
     true
@@ -36,6 +37,7 @@ class FeedManager
 
   def unpush_from_home(account, status)
     return false unless remove_from_feed(:home, account.id, status, account.user&.aggregates_reblogs?)
+
     redis.publish("timeline:#{account.id}", Oj.dump(event: :delete, payload: status.id.to_s))
     true
   end
@@ -46,7 +48,9 @@ class FeedManager
       should_filter &&= !ListAccount.where(list_id: list.id, account_id: status.in_reply_to_account_id).exists?
       return false if should_filter
     end
+
     return false unless add_to_feed(:list, list.id, status, list.account.user&.aggregates_reblogs?)
+
     trim(:list, list.id)
     PushUpdateWorker.perform_async(list.account_id, status.id, "timeline:list:#{list.id}") if push_update_required?("timeline:list:#{list.id}")
     true
@@ -54,6 +58,7 @@ class FeedManager
 
   def unpush_from_list(list, status)
     return false unless remove_from_feed(:list, list.id, status, list.account.user&.aggregates_reblogs?)
+
     redis.publish("timeline:list:#{list.id}", Oj.dump(event: :delete, payload: status.id.to_s))
     true
   end
@@ -85,16 +90,21 @@ class FeedManager
 
   def merge_into_timeline(from_account, into_account)
     timeline_key = key(:home, into_account.id)
-    query        = from_account.statuses.limit(FeedManager::MAX_ITEMS / 4)
+    aggregate    = into_account.user&.aggregates_reblogs?
+    query        = from_account.statuses.where(visibility: [:public, :unlisted, :private]).includes(:preloadable_poll, reblog: :account).limit(FeedManager::MAX_ITEMS / 4)
 
     if redis.zcard(timeline_key) >= FeedManager::MAX_ITEMS / 4
-      oldest_home_score = redis.zrange(timeline_key, 0, 0, with_scores: true)&.first&.last&.to_i || 0
+      oldest_home_score = redis.zrange(timeline_key, 0, 0, with_scores: true).first.last.to_i
       query = query.where('id > ?', oldest_home_score)
     end
 
-    query.each do |status|
-      next if status.direct_visibility? || status.limited_visibility? || filter?(:home, status, into_account)
-      add_to_feed(:home, into_account.id, status, into_account.user&.aggregates_reblogs?)
+    statuses = query.to_a
+    crutches = build_crutches(into_account.id, statuses)
+
+    statuses.each do |status|
+      next if filter_from_home?(status, into_account, crutches)
+
+      add_to_feed(:home, into_account.id, status, aggregate)
     end
 
     trim(:home, into_account.id)
@@ -120,24 +130,35 @@ class FeedManager
   end
 
   def populate_feed(account)
-    added  = 0
-    limit  = FeedManager::MAX_ITEMS / 2
-    max_id = nil
+    limit        = FeedManager::MAX_ITEMS / 2
+    aggregate    = account.user&.aggregates_reblogs?
+    timeline_key = key(:home, account.id)
 
-    loop do
-      statuses = Status.as_home_timeline(account)
-                       .paginate_by_max_id(limit, max_id)
+    account.statuses.where.not(visibility: :direct).limit(limit).each do |status|
+      add_to_feed(:home, account.id, status, aggregate)
+    end
 
-      break if statuses.empty?
+    account.following.includes(:account_stat).find_each do |target_account|
+      if redis.zcard(timeline_key) >= limit
+        oldest_home_score = redis.zrange(timeline_key, 0, 0, with_scores: true).first.last.to_i
+        last_status_score = Mastodon::Snowflake.id_at(account.last_status_at)
 
-      statuses.each do |status|
-        next if filter_from_home?(status, account)
-        added += 1 if add_to_feed(:home, account.id, status, account.user&.aggregates_reblogs?)
+        # If the feed is full and this account has not posted more recently
+        # than the last item on the feed, then we can skip the whole account
+        # because none of its statuses would stay on the feed anyway
+        next if last_status_score < oldest_home_score
       end
 
-      break unless added.zero?
+      statuses = target_account.statuses.where(visibility: [:public, :unlisted, :private]).includes(:preloadable_poll, reblog: :account).limit(limit)
+      crutches = build_crutches(account.id, statuses)
+
+      statuses.each do |status|
+        next if filter_from_home?(status, account, crutches)
+
+        add_to_feed(:home, account.id, status, aggregate)
+      end
 
-      max_id = statuses.last.id
+      trim(:home, account.id)
     end
   end
 
@@ -152,31 +173,33 @@ class FeedManager
       (context == :home ? Mute.where(account_id: receiver_id, target_account_id: account_ids).any? : Mute.where(account_id: receiver_id, target_account_id: account_ids, hide_notifications: true).any?)
   end
 
-  def filter_from_home?(status, receiver_id)
+  def filter_from_home?(status, receiver_id, crutches)
     return false if receiver_id == status.account_id
     return true  if status.reply? && (status.in_reply_to_id.nil? || status.in_reply_to_account_id.nil?)
     return true  if phrase_filtered?(status, receiver_id, :home)
 
-    check_for_blocks = status.active_mentions.pluck(:account_id)
+    check_for_blocks = crutches[:active_mentions][status.id] || []
     check_for_blocks.concat([status.account_id])
 
     if status.reblog?
       check_for_blocks.concat([status.reblog.account_id])
-      check_for_blocks.concat(status.reblog.active_mentions.pluck(:account_id))
+      check_for_blocks.concat(crutches[:active_mentions][status.reblog_of_id] || [])
     end
 
-    return true if blocks_or_mutes?(receiver_id, check_for_blocks, :home)
+    return true if check_for_blocks.any? { |target_account_id| crutches[:blocking][target_account_id] || crutches[:muting][target_account_id] }
 
     if status.reply? && !status.in_reply_to_account_id.nil?                                                                      # Filter out if it's a reply
-      should_filter   = !Follow.where(account_id: receiver_id, target_account_id: status.in_reply_to_account_id).exists?         # and I'm not following the person it's a reply to
+      should_filter   = !crutches[:following][status.in_reply_to_account_id]                                                     # and I'm not following the person it's a reply to
       should_filter &&= receiver_id != status.in_reply_to_account_id                                                             # and it's not a reply to me
       should_filter &&= status.account_id != status.in_reply_to_account_id                                                       # and it's not a self-reply
-      return should_filter
+
+      return !!should_filter
     elsif status.reblog?                                                                                                         # Filter out a reblog
-      should_filter   = Follow.where(account_id: receiver_id, target_account_id: status.account_id, show_reblogs: false).exists? # if the reblogger's reblogs are suppressed
-      should_filter ||= Block.where(account_id: status.reblog.account_id, target_account_id: receiver_id).exists?                # or if the author of the reblogged status is blocking me
-      should_filter ||= AccountDomainBlock.where(account_id: receiver_id, domain: status.reblog.account.domain).exists?          # or the author's domain is blocked
-      return should_filter
+      should_filter   = crutches[:hiding_reblogs][status.account_id]                                                             # if the reblogger's reblogs are suppressed
+      should_filter ||= crutches[:blocked_by][status.reblog.account_id]                                                          # or if the author of the reblogged status is blocking me
+      should_filter ||= crutches[:domain_blocking][status.reblog.account.domain]                                                 # or the author's domain is blocked
+
+      return !!should_filter
     end
 
     false
@@ -308,4 +331,31 @@ class FeedManager
 
     redis.zrem(timeline_key, status.id)
   end
+
+  def build_crutches(receiver_id, statuses)
+    crutches = {}
+
+    crutches[:active_mentions] = Mention.active.where(status_id: statuses.flat_map { |s| [s.id, s.reblog_of_id] }.compact).pluck(:status_id, :account_id).each_with_object({}) { |(id, account_id), mapping| (mapping[id] ||= []).push(account_id) }
+
+    check_for_blocks = statuses.flat_map do |s|
+      arr = crutches[:active_mentions][s.id] || []
+      arr.concat([s.account_id])
+
+      if s.reblog?
+        arr.concat([s.reblog.account_id])
+        arr.concat(crutches[:active_mentions][s.reblog_of_id] || [])
+      end
+
+      arr
+    end
+
+    crutches[:following]       = Follow.where(account_id: receiver_id, target_account_id: statuses.map(&:in_reply_to_account_id).compact).pluck(:target_account_id).each_with_object({}) { |id, mapping| mapping[id] = true }
+    crutches[:hiding_reblogs]  = Follow.where(account_id: receiver_id, target_account_id: statuses.map { |s| s.account_id if s.reblog? }.compact, show_reblogs: false).pluck(:target_account_id).each_with_object({}) { |id, mapping| mapping[id] = true }
+    crutches[:blocking]        = Block.where(account_id: receiver_id, target_account_id: check_for_blocks).pluck(:target_account_id).each_with_object({}) { |id, mapping| mapping[id] = true }
+    crutches[:muting]          = Mute.where(account_id: receiver_id, target_account_id: check_for_blocks).pluck(:target_account_id).each_with_object({}) { |id, mapping| mapping[id] = true }
+    crutches[:domain_blocking] = AccountDomainBlock.where(account_id: receiver_id, domain: statuses.map { |s| s.reblog&.account&.domain }.compact).pluck(:domain).each_with_object({}) { |domain, mapping| mapping[domain] = true }
+    crutches[:blocked_by]      = Block.where(target_account_id: receiver_id, account_id: statuses.map { |s| s.reblog&.account_id }.compact).pluck(:account_id).each_with_object({}) { |id, mapping| mapping[id] = true }
+
+    crutches
+  end
 end
diff --git a/app/models/home_feed.rb b/app/models/home_feed.rb
index ba7564983..1fd506138 100644
--- a/app/models/home_feed.rb
+++ b/app/models/home_feed.rb
@@ -7,19 +7,7 @@ class HomeFeed < Feed
     @account = account
   end
 
-  def get(limit, max_id = nil, since_id = nil, min_id = nil)
-    if redis.exists("account:#{@account.id}:regeneration")
-      from_database(limit, max_id, since_id, min_id)
-    else
-      super
-    end
-  end
-
-  private
-
-  def from_database(limit, max_id, since_id, min_id)
-    Status.as_home_timeline(@account)
-          .paginate_by_id(limit, max_id: max_id, since_id: since_id, min_id: min_id)
-          .reject { |status| FeedManager.instance.filter?(:home, status, @account.id) }
+  def regenerating?
+    redis.exists("account:#{@id}:regeneration")
   end
 end
diff --git a/app/models/status.rb b/app/models/status.rb
index 3504ac370..0c01a5389 100644
--- a/app/models/status.rb
+++ b/app/models/status.rb
@@ -282,10 +282,6 @@ class Status < ApplicationRecord
       where(language: nil).or where(language: account.chosen_languages)
     end
 
-    def as_home_timeline(account)
-      where(account: [account] + account.following).where(visibility: [:public, :unlisted, :private])
-    end
-
     def as_public_timeline(account = nil, local_only = false)
       query = timeline_scope(local_only).without_replies
 
diff --git a/spec/models/home_feed_spec.rb b/spec/models/home_feed_spec.rb
index 3acb997f1..ee7a83960 100644
--- a/spec/models/home_feed_spec.rb
+++ b/spec/models/home_feed_spec.rb
@@ -34,11 +34,10 @@ RSpec.describe HomeFeed, type: :model do
         Redis.current.set("account:#{account.id}:regeneration", true)
       end
 
-      it 'gets statuses with ids in the range from database' do
+      it 'returns nothing' do
         results = subject.get(3)
 
-        expect(results.map(&:id)).to eq [10, 3, 2]
-        expect(results.first.attributes.keys).to include('id', 'updated_at')
+        expect(results.map(&:id)).to eq []
       end
     end
   end
diff --git a/spec/models/status_spec.rb b/spec/models/status_spec.rb
index a8c567414..51a10cd17 100644
--- a/spec/models/status_spec.rb
+++ b/spec/models/status_spec.rb
@@ -296,49 +296,6 @@ RSpec.describe Status, type: :model do
     end
   end
 
-  describe '.as_home_timeline' do
-    let(:account) { Fabricate(:account) }
-    let(:followed) { Fabricate(:account) }
-    let(:not_followed) { Fabricate(:account) }
-
-    before do
-      Fabricate(:follow, account: account, target_account: followed)
-
-      @self_status = Fabricate(:status, account: account, visibility: :public)
-      @self_direct_status = Fabricate(:status, account: account, visibility: :direct)
-      @followed_status = Fabricate(:status, account: followed, visibility: :public)
-      @followed_direct_status = Fabricate(:status, account: followed, visibility: :direct)
-      @not_followed_status = Fabricate(:status, account: not_followed, visibility: :public)
-
-      @results = Status.as_home_timeline(account)
-    end
-
-    it 'includes statuses from self' do
-      expect(@results).to include(@self_status)
-    end
-
-    it 'does not include direct statuses from self' do
-      expect(@results).to_not include(@self_direct_status)
-    end
-
-    it 'includes statuses from followed' do
-      expect(@results).to include(@followed_status)
-    end
-
-    it 'does not include direct statuses mentioning recipient from followed' do
-      Fabricate(:mention, account: account, status: @followed_direct_status)
-      expect(@results).to_not include(@followed_direct_status)
-    end
-
-    it 'does not include direct statuses not mentioning recipient from followed' do
-      expect(@results).not_to include(@followed_direct_status)
-    end
-
-    it 'does not include statuses from non-followed' do
-      expect(@results).not_to include(@not_followed_status)
-    end
-  end
-
   describe '.as_public_timeline' do
     it 'only includes statuses with public visibility' do
       public_status = Fabricate(:status, visibility: :public)
-- 
cgit 


From c8bcf5cbfdc4b076eae0d9091e688436aa7f2508 Mon Sep 17 00:00:00 2001
From: Eugen Rochko <eugen@zeonfederated.com>
Date: Wed, 9 Oct 2019 00:30:15 +0200
Subject: Add admin setting to auto-approve hashtags (#12122)

Change inaccurate labels on other admin settings
---
 app/models/form/admin_settings.rb       |  2 ++
 app/models/tag.rb                       |  2 +-
 app/views/admin/settings/edit.html.haml |  9 ++++++---
 config/locales/en.yml                   | 11 +++++++----
 config/settings.yml                     |  1 +
 5 files changed, 17 insertions(+), 8 deletions(-)

(limited to 'app/models')

diff --git a/app/models/form/admin_settings.rb b/app/models/form/admin_settings.rb
index 24196e182..70e9c21f1 100644
--- a/app/models/form/admin_settings.rb
+++ b/app/models/form/admin_settings.rb
@@ -30,6 +30,7 @@ class Form::AdminSettings
     mascot
     spam_check_enabled
     trends
+    trendable_by_default
     show_domain_blocks
     show_domain_blocks_rationale
     noindex
@@ -46,6 +47,7 @@ class Form::AdminSettings
     profile_directory
     spam_check_enabled
     trends
+    trendable_by_default
     noindex
   ).freeze
 
diff --git a/app/models/tag.rb b/app/models/tag.rb
index 82786daa8..59445a83b 100644
--- a/app/models/tag.rb
+++ b/app/models/tag.rb
@@ -76,7 +76,7 @@ class Tag < ApplicationRecord
   alias listable? listable
 
   def trendable
-    boolean_with_default('trendable', false)
+    boolean_with_default('trendable', Setting.trendable_by_default)
   end
 
   alias trendable? trendable
diff --git a/app/views/admin/settings/edit.html.haml b/app/views/admin/settings/edit.html.haml
index 752386b3c..6282bb39c 100644
--- a/app/views/admin/settings/edit.html.haml
+++ b/app/views/admin/settings/edit.html.haml
@@ -20,10 +20,10 @@
       = f.input :site_contact_email, wrapper: :with_label, label: t('admin.settings.contact_information.email')
 
   .fields-group
-    = f.input :site_description, wrapper: :with_block_label, as: :text, label: t('admin.settings.site_description.title'), hint: t('admin.settings.site_description.desc_html'), input_html: { rows: 4 }
+    = f.input :site_short_description, wrapper: :with_block_label, as: :text, label: t('admin.settings.site_short_description.title'), hint: t('admin.settings.site_short_description.desc_html'), input_html: { rows: 2 }
 
   .fields-group
-    = f.input :site_short_description, wrapper: :with_block_label, as: :text, label: t('admin.settings.site_short_description.title'), hint: t('admin.settings.site_short_description.desc_html'), input_html: { rows: 2 }
+    = f.input :site_description, wrapper: :with_block_label, as: :text, label: t('admin.settings.site_description.title'), hint: t('admin.settings.site_description.desc_html'), input_html: { rows: 2 }
 
   .fields-row
     .fields-row__column.fields-row__column-6.fields-group
@@ -71,6 +71,9 @@
     .fields-group
       = f.input :trends, as: :boolean, wrapper: :with_label, label: t('admin.settings.trends.title'), hint: t('admin.settings.trends.desc_html')
 
+    .fields-group
+      = f.input :trendable_by_default, as: :boolean, wrapper: :with_label, label: t('admin.settings.trendable_by_default.title'), hint: t('admin.settings.trendable_by_default.desc_html')
+
     .fields-group
       = f.input :noindex, as: :boolean, wrapper: :with_label, label: t('admin.settings.default_noindex.title'), hint: t('admin.settings.default_noindex.desc_html')
 
@@ -89,8 +92,8 @@
       = f.input :show_domain_blocks_rationale, wrapper: :with_label, collection: %i(disabled users all), label: t('admin.settings.domain_blocks_rationale.title'), label_method: lambda { |value| t("admin.settings.domain_blocks.#{value}") }, include_blank: false, collection_wrapper_tag: 'ul', item_wrapper_tag: 'li'
 
   .fields-group
-    = f.input :closed_registrations_message, as: :text, wrapper: :with_block_label, label: t('admin.settings.registrations.closed_message.title'), hint: t('admin.settings.registrations.closed_message.desc_html'), input_html: { rows: 8 }
     = f.input :site_extended_description, wrapper: :with_block_label, as: :text, label: t('admin.settings.site_description_extended.title'), hint: t('admin.settings.site_description_extended.desc_html'), input_html: { rows: 8 } unless whitelist_mode?
+    = f.input :closed_registrations_message, as: :text, wrapper: :with_block_label, label: t('admin.settings.registrations.closed_message.title'), hint: t('admin.settings.registrations.closed_message.desc_html'), input_html: { rows: 8 }
     = f.input :site_terms, wrapper: :with_block_label, as: :text, label: t('admin.settings.site_terms.title'), hint: t('admin.settings.site_terms.desc_html'), input_html: { rows: 8 }
     = f.input :custom_css, wrapper: :with_block_label, as: :text, input_html: { rows: 8 }, label: t('admin.settings.custom_css.title'), hint: t('admin.settings.custom_css.desc_html')
 
diff --git a/config/locales/en.yml b/config/locales/en.yml
index 68fc21323..0e8ee6a76 100644
--- a/config/locales/en.yml
+++ b/config/locales/en.yml
@@ -478,8 +478,8 @@ en:
           open: Anyone can sign up
         title: Registrations mode
       show_known_fediverse_at_about_page:
-        desc_html: When toggled, it will show toots from all the known fediverse on preview. Otherwise it will only show local toots.
-        title: Show known fediverse on timeline preview
+        desc_html: When disabled, restricts the public timeline linked from the landing page to showing only local content
+        title: Include federated content on unauthenticated public timeline page
       show_staff_badge:
         desc_html: Show a staff badge on a user page
         title: Show staff badge
@@ -503,9 +503,12 @@ en:
         desc_html: Used for previews via OpenGraph and API. 1200x630px recommended
         title: Server thumbnail
       timeline_preview:
-        desc_html: Display public timeline on landing page
-        title: Timeline preview
+        desc_html: Display link to public timeline on landing page and allow API access to the public timeline without authentication
+        title: Allow unauthenticated access to public timeline
       title: Site settings
+      trendable_by_default:
+        desc_html: Affects hashtags that have not been previously disallowed
+        title: Allow hashtags to trend without prior review
       trends:
         desc_html: Publicly display previously reviewed hashtags that are currently trending
         title: Trending hashtags
diff --git a/config/settings.yml b/config/settings.yml
index 6dbc46706..bd2f65b5e 100644
--- a/config/settings.yml
+++ b/config/settings.yml
@@ -35,6 +35,7 @@ defaults: &defaults
   use_blurhash: true
   use_pending_items: false
   trends: true
+  trendable_by_default: false
   notification_emails:
     follow: false
     reblog: false
-- 
cgit 


From 354fdd317e9c495ed721013911bc5274d5e0e1f8 Mon Sep 17 00:00:00 2001
From: Eugen Rochko <eugen@zeonfederated.com>
Date: Wed, 9 Oct 2019 07:10:46 +0200
Subject: Fix attachment not being re-downloaded even if file is not stored
 (#12125)

Change the behaviour of remotable concern. Previously, it would skip
downloading an attachment if the stored remote URL is identical to
the new one. Now it would not be skipped if the attachment is not
actually currently stored by Paperclip.
---
 app/controllers/api/v1/streaming_controller.rb | 14 ++++++++++----
 app/models/account.rb                          |  7 +++----
 app/models/concerns/remotable.rb               |  2 +-
 config/initializers/paperclip.rb               | 19 +++++++++++++------
 spec/models/account_spec.rb                    |  4 ++--
 spec/models/concerns/remotable_spec.rb         | 13 ++++++++++++-
 6 files changed, 41 insertions(+), 18 deletions(-)

(limited to 'app/models')

diff --git a/app/controllers/api/v1/streaming_controller.rb b/app/controllers/api/v1/streaming_controller.rb
index 66b812e76..ebb17608c 100644
--- a/app/controllers/api/v1/streaming_controller.rb
+++ b/app/controllers/api/v1/streaming_controller.rb
@@ -5,11 +5,17 @@ class Api::V1::StreamingController < Api::BaseController
 
   def index
     if Rails.configuration.x.streaming_api_base_url != request.host
-      uri = URI.parse(request.url)
-      uri.host = URI.parse(Rails.configuration.x.streaming_api_base_url).host
-      redirect_to uri.to_s, status: 301
+      redirect_to streaming_api_url, status: 301
     else
-      raise ActiveRecord::RecordNotFound
+      not_found
     end
   end
+
+  private
+
+  def streaming_api_url
+    Addressable::URI.parse(request.url).tap do |uri|
+      uri.host = Addressable::URI.parse(Rails.configuration.x.streaming_api_base_url).host
+    end.to_s
+  end
 end
diff --git a/app/models/account.rb b/app/models/account.rb
index 01d45e36c..2f43f337f 100644
--- a/app/models/account.rb
+++ b/app/models/account.rb
@@ -310,10 +310,9 @@ class Account < ApplicationRecord
   def save_with_optional_media!
     save!
   rescue ActiveRecord::RecordInvalid
-    self.avatar              = nil
-    self.header              = nil
-    self[:avatar_remote_url] = ''
-    self[:header_remote_url] = ''
+    self.avatar = nil
+    self.header = nil
+
     save!
   end
 
diff --git a/app/models/concerns/remotable.rb b/app/models/concerns/remotable.rb
index 082302619..b7a476c87 100644
--- a/app/models/concerns/remotable.rb
+++ b/app/models/concerns/remotable.rb
@@ -18,7 +18,7 @@ module Remotable
           return
         end
 
-        return if !%w(http https).include?(parsed_url.scheme) || parsed_url.host.blank? || self[attribute_name] == url
+        return if !%w(http https).include?(parsed_url.scheme) || parsed_url.host.blank? || (self[attribute_name] == url && send("#{attachment_name}_file_name").present?)
 
         begin
           Request.new(:get, url).perform do |response|
diff --git a/config/initializers/paperclip.rb b/config/initializers/paperclip.rb
index a0253f4bc..d3602e655 100644
--- a/config/initializers/paperclip.rb
+++ b/config/initializers/paperclip.rb
@@ -1,10 +1,11 @@
 # frozen_string_literal: true
 
-Paperclip.options[:read_timeout] = 60
-
 Paperclip.interpolates :filename do |attachment, style|
-  return attachment.original_filename if style == :original
-  [basename(attachment, style), extension(attachment, style)].delete_if(&:blank?).join('.')
+  if style == :original
+    attachment.original_filename
+  else
+    [basename(attachment, style), extension(attachment, style)].delete_if(&:blank?).join('.')
+  end
 end
 
 Paperclip::Attachment.default_options.merge!(
@@ -24,17 +25,21 @@ if ENV['S3_ENABLED'] == 'true'
     storage: :s3,
     s3_protocol: s3_protocol,
     s3_host_name: s3_hostname,
+
     s3_headers: {
       'X-Amz-Multipart-Threshold' => ENV.fetch('S3_MULTIPART_THRESHOLD') { 15.megabytes }.to_i,
       'Cache-Control' => 'public, max-age=315576000, immutable',
     },
+
     s3_permissions: ENV.fetch('S3_PERMISSION') { 'public-read' },
     s3_region: s3_region,
+
     s3_credentials: {
       bucket: ENV['S3_BUCKET'],
       access_key_id: ENV['AWS_ACCESS_KEY_ID'],
       secret_access_key: ENV['AWS_SECRET_ACCESS_KEY'],
     },
+
     s3_options: {
       signature_version: ENV.fetch('S3_SIGNATURE_VERSION') { 'v4' },
       http_open_timeout: 5,
@@ -49,6 +54,7 @@ if ENV['S3_ENABLED'] == 'true'
       endpoint: ENV['S3_ENDPOINT'],
       force_path_style: true
     )
+
     Paperclip::Attachment.default_options[:url] = ':s3_path_url'
   end
 
@@ -74,6 +80,7 @@ elsif ENV['SWIFT_ENABLED'] == 'true'
       openstack_region: ENV['SWIFT_REGION'],
       openstack_cache_ttl: ENV.fetch('SWIFT_CACHE_TTL') { 60 },
     },
+
     fog_directory: ENV['SWIFT_CONTAINER'],
     fog_host: ENV['SWIFT_OBJECT_URL'],
     fog_public: true
@@ -82,7 +89,7 @@ else
   Paperclip::Attachment.default_options.merge!(
     storage: :filesystem,
     use_timestamp: true,
-    path: (ENV['PAPERCLIP_ROOT_PATH'] || ':rails_root/public/system') + '/:class/:attachment/:id_partition/:style/:filename',
-    url: (ENV['PAPERCLIP_ROOT_URL'] || '/system') + '/:class/:attachment/:id_partition/:style/:filename',
+    path: ENV.fetch('PAPERCLIP_ROOT_PATH', ':rails_root/public/system') + '/:class/:attachment/:id_partition/:style/:filename',
+    url: ENV.fetch('PAPERCLIP_ROOT_URL', '/system') + '/:class/:attachment/:id_partition/:style/:filename',
   )
 end
diff --git a/spec/models/account_spec.rb b/spec/models/account_spec.rb
index 3eec464bd..b2f6234cb 100644
--- a/spec/models/account_spec.rb
+++ b/spec/models/account_spec.rb
@@ -126,8 +126,8 @@ RSpec.describe Account, type: :model do
       end
 
       it 'sets default avatar, header, avatar_remote_url, and header_remote_url' do
-        expect(account.avatar_remote_url).to eq ''
-        expect(account.header_remote_url).to eq ''
+        expect(account.avatar_remote_url).to eq 'https://remote.test/invalid_avatar'
+        expect(account.header_remote_url).to eq expectation.header_remote_url
         expect(account.avatar_file_name).to  eq nil
         expect(account.header_file_name).to  eq nil
       end
diff --git a/spec/models/concerns/remotable_spec.rb b/spec/models/concerns/remotable_spec.rb
index a4289cc45..99a60cbf6 100644
--- a/spec/models/concerns/remotable_spec.rb
+++ b/spec/models/concerns/remotable_spec.rb
@@ -18,6 +18,8 @@ RSpec.describe Remotable do
 
     def hoge=(arg); end
 
+    def hoge_file_name; end
+
     def hoge_file_name=(arg); end
 
     def has_attribute?(arg); end
@@ -109,12 +111,21 @@ RSpec.describe Remotable do
       end
 
       context 'foo[attribute_name] == url' do
-        it 'makes no request' do
+        it 'makes no request if file is saved' do
           allow(foo).to receive(:[]).with(attribute_name).and_return(url)
+          allow(foo).to receive(:hoge_file_name).and_return('foo.jpg')
 
           foo.hoge_remote_url = url
           expect(request).not_to have_been_requested
         end
+
+        it 'makes request if file is not saved' do
+          allow(foo).to receive(:[]).with(attribute_name).and_return(url)
+          allow(foo).to receive(:hoge_file_name).and_return(nil)
+
+          foo.hoge_remote_url = url
+          expect(request).to have_been_requested
+        end
       end
 
       context "scheme is https, parsed_url.host isn't empty, and foo[attribute_name] != url" do
-- 
cgit 


From b5f7e12817356b9b1795ab0187fe08d07f13a485 Mon Sep 17 00:00:00 2001
From: Eugen Rochko <eugen@zeonfederated.com>
Date: Wed, 9 Oct 2019 07:11:23 +0200
Subject: Remove auto-silence behaviour from spam check (#12117)

Fix #12113
---
 app/lib/spam_check.rb              |  7 +------
 app/models/account.rb              |  2 +-
 app/models/admin/account_action.rb | 12 ++++++++++++
 config/locales/en.yml              |  2 +-
 spec/lib/spam_check_spec.rb        |  4 ----
 5 files changed, 15 insertions(+), 12 deletions(-)

(limited to 'app/models')

diff --git a/app/lib/spam_check.rb b/app/lib/spam_check.rb
index 441697364..235e44230 100644
--- a/app/lib/spam_check.rb
+++ b/app/lib/spam_check.rb
@@ -44,7 +44,6 @@ class SpamCheck
   end
 
   def flag!
-    auto_silence_account!
     auto_report_status!
   end
 
@@ -134,17 +133,13 @@ class SpamCheck
     text.gsub(/\s+/, ' ').strip
   end
 
-  def auto_silence_account!
-    @account.silence!
-  end
-
   def auto_report_status!
     status_ids = Status.where(visibility: %i(public unlisted)).where(id: matching_status_ids).pluck(:id) + [@status.id] if @status.distributable?
     ReportService.new.call(Account.representative, @account, status_ids: status_ids, comment: I18n.t('spam_check.spam_detected_and_silenced'))
   end
 
   def already_flagged?
-    @account.silenced?
+    @account.silenced? || @account.targeted_reports.unresolved.where(account_id: -99).exists?
   end
 
   def trusted?
diff --git a/app/models/account.rb b/app/models/account.rb
index 2f43f337f..05936def3 100644
--- a/app/models/account.rb
+++ b/app/models/account.rb
@@ -198,7 +198,7 @@ class Account < ApplicationRecord
   end
 
   def unsilence!
-    update!(silenced_at: nil, trust_level: trust_level == TRUST_LEVELS[:untrusted] ? TRUST_LEVELS[:trusted] : trust_level)
+    update!(silenced_at: nil)
   end
 
   def suspended?
diff --git a/app/models/admin/account_action.rb b/app/models/admin/account_action.rb
index b30a82369..e9da003a3 100644
--- a/app/models/admin/account_action.rb
+++ b/app/models/admin/account_action.rb
@@ -62,6 +62,8 @@ class Admin::AccountAction
 
   def process_action!
     case type
+    when 'none'
+      handle_resolve!
     when 'disable'
       handle_disable!
     when 'silence'
@@ -103,6 +105,16 @@ class Admin::AccountAction
     end
   end
 
+  def handle_resolve!
+    if with_report? && report.account_id == -99 && target_account.trust_level == Account::TRUST_LEVELS[:untrusted]
+      # This is an automated report and it is being dismissed, so it's
+      # a false positive, in which case update the account's trust level
+      # to prevent further spam checks
+
+      target_account.update(trust_level: Account::TRUST_LEVELS[:trusted])
+    end
+  end
+
   def handle_disable!
     authorize(target_account.user, :disable?)
     log_action(:disable, target_account.user)
diff --git a/config/locales/en.yml b/config/locales/en.yml
index 0e8ee6a76..1ffc99eb3 100644
--- a/config/locales/en.yml
+++ b/config/locales/en.yml
@@ -497,7 +497,7 @@ en:
         title: Custom terms of service
       site_title: Server name
       spam_check_enabled:
-        desc_html: Mastodon can auto-silence and auto-report accounts that send repeated unsolicited messages. There may be false positives.
+        desc_html: Mastodon can auto-report accounts that send repeated unsolicited messages. There may be false positives.
         title: Anti-spam automation
       thumbnail:
         desc_html: Used for previews via OpenGraph and API. 1200x630px recommended
diff --git a/spec/lib/spam_check_spec.rb b/spec/lib/spam_check_spec.rb
index 4cae46111..d4d66a499 100644
--- a/spec/lib/spam_check_spec.rb
+++ b/spec/lib/spam_check_spec.rb
@@ -181,10 +181,6 @@ RSpec.describe SpamCheck do
       described_class.new(status2).flag!
     end
 
-    it 'silences the account' do
-      expect(sender.silenced?).to be true
-    end
-
     it 'creates a report about the account' do
       expect(sender.targeted_reports.unresolved.count).to eq 1
     end
-- 
cgit 


From 915f3712ae7ae44c0cbe50c9694c25e3ee87a540 Mon Sep 17 00:00:00 2001
From: Eugen Rochko <eugen@zeonfederated.com>
Date: Thu, 10 Oct 2019 02:22:04 +0200
Subject: Fix admin setting to auto-approve hashtags not affecting query
 (#12130)

Follow-up to #12122
---
 app/models/tag.rb           | 1 +
 app/models/trending_tags.rb | 2 +-
 2 files changed, 2 insertions(+), 1 deletion(-)

(limited to 'app/models')

diff --git a/app/models/tag.rb b/app/models/tag.rb
index 59445a83b..d3a7e1e6d 100644
--- a/app/models/tag.rb
+++ b/app/models/tag.rb
@@ -37,6 +37,7 @@ class Tag < ApplicationRecord
   scope :pending_review, -> { unreviewed.where.not(requested_review_at: nil) }
   scope :usable, -> { where(usable: [true, nil]) }
   scope :listable, -> { where(listable: [true, nil]) }
+  scope :trendable, -> { Setting.trendable_by_default ? where(trendable: [true, nil]) : where(trendable: true) }
   scope :discoverable, -> { listable.joins(:account_tag_stat).where(AccountTagStat.arel_table[:accounts_count].gt(0)).order(Arel.sql('account_tag_stats.accounts_count desc')) }
   scope :most_used, ->(account) { joins(:statuses).where(statuses: { account: account }).group(:id).order(Arel.sql('count(*) desc')) }
   scope :matches_name, ->(value) { where(arel_table[:name].matches("#{value}%")) }
diff --git a/app/models/trending_tags.rb b/app/models/trending_tags.rb
index 8cdade42d..c69f6d3c3 100644
--- a/app/models/trending_tags.rb
+++ b/app/models/trending_tags.rb
@@ -90,7 +90,7 @@ class TrendingTags
       tag_ids = redis.zrevrange(KEY, 0, LIMIT - 1).map(&:to_i)
 
       tags = Tag.where(id: tag_ids)
-      tags = tags.where(trendable: true) if filtered
+      tags = tags.trendable if filtered
       tags = tags.each_with_object({}) { |tag, h| h[tag.id] = tag }
 
       tag_ids.map { |tag_id| tags[tag_id] }.compact.take(limit)
-- 
cgit 


From 488dd0ff7a41584424f871841457affae11390e5 Mon Sep 17 00:00:00 2001
From: Hugo Gameiro <hmgameiro@gmail.com>
Date: Thu, 24 Oct 2019 21:47:58 +0100
Subject: remove audio metadata (#12171)

---
 app/models/media_attachment.rb | 1 +
 1 file changed, 1 insertion(+)

(limited to 'app/models')

diff --git a/app/models/media_attachment.rb b/app/models/media_attachment.rb
index 9c6c04556..137377422 100644
--- a/app/models/media_attachment.rb
+++ b/app/models/media_attachment.rb
@@ -87,6 +87,7 @@ class MediaAttachment < ApplicationRecord
       convert_options: {
         output: {
           'loglevel' => 'fatal',
+          'map_metadata' => '-1',
           'q:a' => 2,
         },
       },
-- 
cgit 


From 48f75b86aed816ef5afaaae64416dbeaa14e4fda Mon Sep 17 00:00:00 2001
From: Faye Duxovni <duxovni@duxovni.org>
Date: Thu, 24 Oct 2019 16:51:41 -0400
Subject: Add setting for whether to crop images in unexpanded toots (#12126)

---
 app/controllers/settings/preferences_controller.rb       |  1 +
 app/javascript/mastodon/components/media_gallery.js      | 16 ++++++++--------
 app/javascript/mastodon/initial_state.js                 |  1 +
 app/lib/user_settings_decorator.rb                       |  5 +++++
 app/models/user.rb                                       |  2 +-
 app/serializers/initial_state_serializer.rb              |  2 ++
 app/views/settings/preferences/appearance/show.html.haml |  5 +++++
 config/locales/en.yml                                    |  1 +
 config/locales/simple_form.en.yml                        |  1 +
 config/settings.yml                                      |  1 +
 10 files changed, 26 insertions(+), 9 deletions(-)

(limited to 'app/models')

diff --git a/app/controllers/settings/preferences_controller.rb b/app/controllers/settings/preferences_controller.rb
index edf29947b..bac9b329d 100644
--- a/app/controllers/settings/preferences_controller.rb
+++ b/app/controllers/settings/preferences_controller.rb
@@ -57,6 +57,7 @@ class Settings::PreferencesController < Settings::BaseController
       :setting_use_blurhash,
       :setting_use_pending_items,
       :setting_trends,
+      :setting_crop_images,
       notification_emails: %i(follow follow_request reblog favourite mention digest report pending_account trending_tag),
       interactions: %i(must_be_follower must_be_following must_be_following_dm)
     )
diff --git a/app/javascript/mastodon/components/media_gallery.js b/app/javascript/mastodon/components/media_gallery.js
index b8fca8bcb..12b7e5b66 100644
--- a/app/javascript/mastodon/components/media_gallery.js
+++ b/app/javascript/mastodon/components/media_gallery.js
@@ -6,7 +6,7 @@ import IconButton from './icon_button';
 import { defineMessages, injectIntl, FormattedMessage } from 'react-intl';
 import { isIOS } from '../is_mobile';
 import classNames from 'classnames';
-import { autoPlayGif, displayMedia, useBlurhash } from '../initial_state';
+import { autoPlayGif, cropImages, displayMedia, useBlurhash } from '../initial_state';
 import { decode } from 'blurhash';
 
 const messages = defineMessages({
@@ -281,7 +281,7 @@ class MediaGallery extends React.PureComponent {
   }
 
   handleRef = (node) => {
-    if (node /*&& this.isStandaloneEligible()*/) {
+    if (node) {
       // offsetWidth triggers a layout, so only calculate when we need to
       if (this.props.cacheWidth) this.props.cacheWidth(node.offsetWidth);
 
@@ -291,13 +291,13 @@ class MediaGallery extends React.PureComponent {
     }
   }
 
-  isStandaloneEligible() {
-    const { media, standalone } = this.props;
-    return standalone && media.size === 1 && media.getIn([0, 'meta', 'small', 'aspect']);
+  isFullSizeEligible() {
+    const { media } = this.props;
+    return media.size === 1 && media.getIn([0, 'meta', 'small', 'aspect']);
   }
 
   render () {
-    const { media, intl, sensitive, height, defaultWidth } = this.props;
+    const { media, intl, sensitive, height, defaultWidth, standalone } = this.props;
     const { visible } = this.state;
 
     const width = this.state.width || defaultWidth;
@@ -306,7 +306,7 @@ class MediaGallery extends React.PureComponent {
 
     const style = {};
 
-    if (this.isStandaloneEligible()) {
+    if (this.isFullSizeEligible() && (standalone || !cropImages)) {
       if (width) {
         style.height = width / this.props.media.getIn([0, 'meta', 'small', 'aspect']);
       }
@@ -319,7 +319,7 @@ class MediaGallery extends React.PureComponent {
     const size     = media.take(4).size;
     const uncached = media.every(attachment => attachment.get('type') === 'unknown');
 
-    if (this.isStandaloneEligible()) {
+    if (standalone && this.isFullSizeEligible()) {
       children = <Item standalone onClick={this.handleClick} attachment={media.get(0)} displayWidth={width} visible={visible} />;
     } else {
       children = media.take(4).map((attachment, i) => <Item key={attachment.get('id')} onClick={this.handleClick} attachment={attachment} index={i} size={size} displayWidth={width} visible={visible || uncached} />);
diff --git a/app/javascript/mastodon/initial_state.js b/app/javascript/mastodon/initial_state.js
index 56fb58546..1134c55db 100644
--- a/app/javascript/mastodon/initial_state.js
+++ b/app/javascript/mastodon/initial_state.js
@@ -24,5 +24,6 @@ export const useBlurhash = getMeta('use_blurhash');
 export const usePendingItems = getMeta('use_pending_items');
 export const showTrends = getMeta('trends');
 export const title = getMeta('title');
+export const cropImages = getMeta('crop_images');
 
 export default initialState;
diff --git a/app/lib/user_settings_decorator.rb b/app/lib/user_settings_decorator.rb
index 3568a3e11..fa8255faa 100644
--- a/app/lib/user_settings_decorator.rb
+++ b/app/lib/user_settings_decorator.rb
@@ -37,6 +37,7 @@ class UserSettingsDecorator
     user.settings['use_blurhash']        = use_blurhash_preference if change?('setting_use_blurhash')
     user.settings['use_pending_items']   = use_pending_items_preference if change?('setting_use_pending_items')
     user.settings['trends']              = trends_preference if change?('setting_trends')
+    user.settings['crop_images']         = crop_images_preference if change?('setting_crop_images')
   end
 
   def merged_notification_emails
@@ -127,6 +128,10 @@ class UserSettingsDecorator
     boolean_cast_setting 'setting_trends'
   end
 
+  def crop_images_preference
+    boolean_cast_setting 'setting_crop_images'
+  end
+
   def boolean_cast_setting(key)
     ActiveModel::Type::Boolean.new.cast(settings[key])
   end
diff --git a/app/models/user.rb b/app/models/user.rb
index 9a19a53b3..7147a9a31 100644
--- a/app/models/user.rb
+++ b/app/models/user.rb
@@ -108,7 +108,7 @@ class User < ApplicationRecord
   delegate :auto_play_gif, :default_sensitive, :unfollow_modal, :boost_modal, :delete_modal,
            :reduce_motion, :system_font_ui, :noindex, :theme, :display_media, :hide_network,
            :expand_spoilers, :default_language, :aggregate_reblogs, :show_application,
-           :advanced_layout, :use_blurhash, :use_pending_items, :trends,
+           :advanced_layout, :use_blurhash, :use_pending_items, :trends, :crop_images,
            to: :settings, prefix: :setting, allow_nil: false
 
   attr_reader :invite_code
diff --git a/app/serializers/initial_state_serializer.rb b/app/serializers/initial_state_serializer.rb
index fb53ea314..392fc891a 100644
--- a/app/serializers/initial_state_serializer.rb
+++ b/app/serializers/initial_state_serializer.rb
@@ -38,11 +38,13 @@ class InitialStateSerializer < ActiveModel::Serializer
       store[:use_pending_items] = object.current_account.user.setting_use_pending_items
       store[:is_staff]          = object.current_account.user.staff?
       store[:trends]            = Setting.trends && object.current_account.user.setting_trends
+      store[:crop_images]       = object.current_account.user.setting_crop_images
     else
       store[:auto_play_gif] = Setting.auto_play_gif
       store[:display_media] = Setting.display_media
       store[:reduce_motion] = Setting.reduce_motion
       store[:use_blurhash]  = Setting.use_blurhash
+      store[:crop_images]   = Setting.crop_images
     end
 
     store
diff --git a/app/views/settings/preferences/appearance/show.html.haml b/app/views/settings/preferences/appearance/show.html.haml
index d6ee1933f..9ed83fb93 100644
--- a/app/views/settings/preferences/appearance/show.html.haml
+++ b/app/views/settings/preferences/appearance/show.html.haml
@@ -25,6 +25,11 @@
     = f.input :setting_reduce_motion, as: :boolean, wrapper: :with_label
     = f.input :setting_system_font_ui, as: :boolean, wrapper: :with_label
 
+  %h4= t 'appearance.toot_layout'
+
+  .fields-group
+    = f.input :setting_crop_images, as: :boolean, wrapper: :with_label
+
   %h4= t 'appearance.discovery'
 
   .fields-group
diff --git a/config/locales/en.yml b/config/locales/en.yml
index 458524c3d..6fc191e1a 100644
--- a/config/locales/en.yml
+++ b/config/locales/en.yml
@@ -576,6 +576,7 @@ en:
     confirmation_dialogs: Confirmation dialogs
     discovery: Discovery
     sensitive_content: Sensitive content
+    toot_layout: Toot layout
   application_mailer:
     notification_preferences: Change e-mail preferences
     salutation: "%{name},"
diff --git a/config/locales/simple_form.en.yml b/config/locales/simple_form.en.yml
index dc39ec926..65951b73b 100644
--- a/config/locales/simple_form.en.yml
+++ b/config/locales/simple_form.en.yml
@@ -113,6 +113,7 @@ en:
         setting_aggregate_reblogs: Group boosts in timelines
         setting_auto_play_gif: Auto-play animated GIFs
         setting_boost_modal: Show confirmation dialog before boosting
+        setting_crop_images: Crop images in non-expanded toots to 16x9
         setting_default_language: Posting language
         setting_default_privacy: Posting privacy
         setting_default_sensitive: Always mark media as sensitive
diff --git a/config/settings.yml b/config/settings.yml
index bd2f65b5e..f66e3922e 100644
--- a/config/settings.yml
+++ b/config/settings.yml
@@ -36,6 +36,7 @@ defaults: &defaults
   use_pending_items: false
   trends: true
   trendable_by_default: false
+  crop_images: true
   notification_emails:
     follow: false
     reblog: false
-- 
cgit 


From 650820d62d46e803083b1178a95ba2a29f7d8b6e Mon Sep 17 00:00:00 2001
From: ThibG <thib@sitedethib.com>
Date: Mon, 4 Nov 2019 13:00:16 +0100
Subject: Fix remote media descriptions being cut off at 420 chars (#12262)

* Fix remote media descriptions being cut off at 420 chars

Fixes #12258

* Fix tests
---
 app/models/media_attachment.rb               |  6 ++++--
 spec/lib/activitypub/activity/create_spec.rb | 26 ++++++++++++++++++++++++++
 spec/models/media_attachment_spec.rb         |  4 ++--
 3 files changed, 32 insertions(+), 4 deletions(-)

(limited to 'app/models')

diff --git a/app/models/media_attachment.rb b/app/models/media_attachment.rb
index 137377422..1f9d92b22 100644
--- a/app/models/media_attachment.rb
+++ b/app/models/media_attachment.rb
@@ -26,6 +26,8 @@ class MediaAttachment < ApplicationRecord
 
   enum type: [:image, :gifv, :video, :unknown, :audio]
 
+  MAX_DESCRIPTION_LENGTH = 1_500
+
   IMAGE_FILE_EXTENSIONS = %w(.jpg .jpeg .png .gif).freeze
   VIDEO_FILE_EXTENSIONS = %w(.webm .mp4 .m4v .mov).freeze
   AUDIO_FILE_EXTENSIONS = %w(.ogg .oga .mp3 .wav .flac .opus .aac .m4a .3gp .wma).freeze
@@ -139,7 +141,7 @@ class MediaAttachment < ApplicationRecord
   include Attachmentable
 
   validates :account, presence: true
-  validates :description, length: { maximum: 1_500 }, if: :local?
+  validates :description, length: { maximum: MAX_DESCRIPTION_LENGTH }, if: :local?
 
   scope :attached,   -> { where.not(status_id: nil).or(where.not(scheduled_status_id: nil)) }
   scope :unattached, -> { where(status_id: nil, scheduled_status_id: nil) }
@@ -243,7 +245,7 @@ class MediaAttachment < ApplicationRecord
   end
 
   def prepare_description
-    self.description = description.strip[0...420] unless description.nil?
+    self.description = description.strip[0...MAX_DESCRIPTION_LENGTH] unless description.nil?
   end
 
   def set_type_and_extension
diff --git a/spec/lib/activitypub/activity/create_spec.rb b/spec/lib/activitypub/activity/create_spec.rb
index 412609de4..b709954a3 100644
--- a/spec/lib/activitypub/activity/create_spec.rb
+++ b/spec/lib/activitypub/activity/create_spec.rb
@@ -261,6 +261,32 @@ RSpec.describe ActivityPub::Activity::Create do
         end
       end
 
+
+      context 'with media attachments with long description' do
+        let(:object_json) do
+          {
+            id: [ActivityPub::TagManager.instance.uri_for(sender), '#bar'].join,
+            type: 'Note',
+            content: 'Lorem ipsum',
+            attachment: [
+              {
+                type: 'Document',
+                mediaType: 'image/png',
+                url: 'http://example.com/attachment.png',
+                name: '*' * 1500,
+              },
+            ],
+          }
+        end
+
+        it 'creates status' do
+          status = sender.statuses.first
+
+          expect(status).to_not be_nil
+          expect(status.media_attachments.map(&:description)).to include('*' * 1500)
+        end
+      end
+
       context 'with media attachments with focal points' do
         let(:object_json) do
           {
diff --git a/spec/models/media_attachment_spec.rb b/spec/models/media_attachment_spec.rb
index 266cd4920..7ddfba7ed 100644
--- a/spec/models/media_attachment_spec.rb
+++ b/spec/models/media_attachment_spec.rb
@@ -136,10 +136,10 @@ RSpec.describe MediaAttachment, type: :model do
   end
 
   describe 'descriptions for remote attachments' do
-    it 'are cut off at 140 characters' do
+    it 'are cut off at 1500 characters' do
       media = Fabricate(:media_attachment, description: 'foo' * 1000, remote_url: 'http://example.com/blah.jpg')
 
-      expect(media.description.size).to be <= 420
+      expect(media.description.size).to be <= 1_500
     end
   end
 end
-- 
cgit 


From 65e13cfacf0d1a862fbc4da44ad84b1522b01b34 Mon Sep 17 00:00:00 2001
From: ThibG <thib@sitedethib.com>
Date: Mon, 4 Nov 2019 13:02:01 +0100
Subject: Add abilityto add oneself to lists (#12271)

* Add ability to add oneself to lists

* Change search results to include oneself when searching through followers

* Mark follow relation as optional in ListAccount
---
 app/models/account.rb                                            | 4 +++-
 app/models/list_account.rb                                       | 6 +++---
 app/services/account_search_service.rb                           | 2 +-
 db/migrate/20191031163205_change_list_account_follow_nullable.rb | 5 +++++
 db/schema.rb                                                     | 4 ++--
 5 files changed, 14 insertions(+), 7 deletions(-)
 create mode 100644 db/migrate/20191031163205_change_list_account_follow_nullable.rb

(limited to 'app/models')

diff --git a/app/models/account.rb b/app/models/account.rb
index 05936def3..cae82da16 100644
--- a/app/models/account.rb
+++ b/app/models/account.rb
@@ -429,6 +429,8 @@ class Account < ApplicationRecord
             SELECT target_account_id
             FROM follows
             WHERE account_id = ?
+            UNION ALL
+            SELECT ?
           )
           SELECT
             accounts.*,
@@ -444,7 +446,7 @@ class Account < ApplicationRecord
           LIMIT ? OFFSET ?
         SQL
 
-        records = find_by_sql([sql, account.id, account.id, account.id, limit, offset])
+        records = find_by_sql([sql, account.id, account.id, account.id, account.id, limit, offset])
       else
         sql = <<-SQL.squish
           SELECT
diff --git a/app/models/list_account.rb b/app/models/list_account.rb
index 87b498224..785923c4c 100644
--- a/app/models/list_account.rb
+++ b/app/models/list_account.rb
@@ -6,13 +6,13 @@
 #  id         :bigint(8)        not null, primary key
 #  list_id    :bigint(8)        not null
 #  account_id :bigint(8)        not null
-#  follow_id  :bigint(8)        not null
+#  follow_id  :bigint(8)
 #
 
 class ListAccount < ApplicationRecord
   belongs_to :list
   belongs_to :account
-  belongs_to :follow
+  belongs_to :follow, optional: true
 
   validates :account_id, uniqueness: { scope: :list_id }
 
@@ -21,6 +21,6 @@ class ListAccount < ApplicationRecord
   private
 
   def set_follow
-    self.follow = Follow.find_by(account_id: list.account_id, target_account_id: account.id)
+    self.follow = Follow.find_by!(account_id: list.account_id, target_account_id: account.id) unless list.account_id == account.id
   end
 end
diff --git a/app/services/account_search_service.rb b/app/services/account_search_service.rb
index 40c5f8590..7e74cc893 100644
--- a/app/services/account_search_service.rb
+++ b/app/services/account_search_service.rb
@@ -127,7 +127,7 @@ class AccountSearchService < BaseService
   end
 
   def following_ids
-    @following_ids ||= account.active_relationships.pluck(:target_account_id)
+    @following_ids ||= account.active_relationships.pluck(:target_account_id) + [account.id]
   end
 
   def limit_for_non_exact_results
diff --git a/db/migrate/20191031163205_change_list_account_follow_nullable.rb b/db/migrate/20191031163205_change_list_account_follow_nullable.rb
new file mode 100644
index 000000000..ff8911546
--- /dev/null
+++ b/db/migrate/20191031163205_change_list_account_follow_nullable.rb
@@ -0,0 +1,5 @@
+class ChangeListAccountFollowNullable < ActiveRecord::Migration[5.1]
+  def change
+    change_column_null :list_accounts, :follow_id, true
+  end
+end
diff --git a/db/schema.rb b/db/schema.rb
index 62e2d3a6d..1bcc003a5 100644
--- a/db/schema.rb
+++ b/db/schema.rb
@@ -10,7 +10,7 @@
 #
 # It's strongly recommended that you check this file into your version control system.
 
-ActiveRecord::Schema.define(version: 2019_10_07_013357) do
+ActiveRecord::Schema.define(version: 2019_10_31_163205) do
 
   # These are extensions that must be enabled in order to support this database
   enable_extension "plpgsql"
@@ -373,7 +373,7 @@ ActiveRecord::Schema.define(version: 2019_10_07_013357) do
   create_table "list_accounts", force: :cascade do |t|
     t.bigint "list_id", null: false
     t.bigint "account_id", null: false
-    t.bigint "follow_id", null: false
+    t.bigint "follow_id"
     t.index ["account_id", "list_id"], name: "index_list_accounts_on_account_id_and_list_id", unique: true
     t.index ["follow_id"], name: "index_list_accounts_on_follow_id"
     t.index ["list_id", "account_id"], name: "index_list_accounts_on_list_id_and_account_id"
-- 
cgit 


From dfea7368c934f600bd0b6b93b4a6c008a4e265b0 Mon Sep 17 00:00:00 2001
From: ThibG <thib@sitedethib.com>
Date: Wed, 13 Nov 2019 23:02:10 +0100
Subject: Add bookmarks (#7107)

* Add backend support for bookmarks

Bookmarks behave like favourites, except they aren't shared with other
users and do not have an associated counter.

* Add spec for bookmark endpoints

* Add front-end support for bookmarks

* Introduce OAuth scopes for bookmarks

* Add bookmarks to archive takeout

* Fix migration

* Coding style fixes

* Fix rebase issue

* Update bookmarked_statuses to latest UI changes

* Update bookmark actions to properly reflect status changes in state

* Add bookmarks item to single-column layout

* Make active bookmarks red
---
 app/controllers/api/v1/bookmarks_controller.rb     |  67 +++++++++++++
 .../api/v1/statuses/bookmarks_controller.rb        |  39 ++++++++
 app/javascript/mastodon/actions/bookmarks.js       |  90 ++++++++++++++++++
 app/javascript/mastodon/actions/interactions.js    |  80 ++++++++++++++++
 .../mastodon/components/status_action_bar.js       |   7 ++
 .../mastodon/containers/status_container.js        |  10 ++
 .../mastodon/features/bookmarked_statuses/index.js | 104 +++++++++++++++++++++
 .../mastodon/features/getting_started/index.js     |   4 +-
 .../features/status/components/action_bar.js       |   7 ++
 app/javascript/mastodon/features/status/index.js   |  11 +++
 .../features/ui/components/columns_area.js         |   2 +
 .../features/ui/components/navigation_panel.js     |   1 +
 app/javascript/mastodon/features/ui/index.js       |   2 +
 .../mastodon/features/ui/util/async-components.js  |   4 +
 app/javascript/mastodon/reducers/status_lists.js   |  29 ++++++
 app/javascript/mastodon/reducers/statuses.js       |   6 ++
 app/javascript/styles/mastodon/components.scss     |   4 +
 app/javascript/styles/mastodon/variables.scss      |   2 +
 app/models/bookmark.rb                             |  26 ++++++
 app/models/concerns/account_associations.rb        |   1 +
 app/models/concerns/account_interactions.rb        |   4 +
 app/models/status.rb                               |   5 +
 app/presenters/status_relationships_presenter.rb   |   2 +
 app/serializers/rest/status_serializer.rb          |   9 ++
 app/services/backup_service.rb                     |  21 +++++
 config/initializers/doorkeeper.rb                  |   2 +
 config/locales/doorkeeper.en.yml                   |   2 +
 config/routes.rb                                   |   4 +
 db/migrate/20180831171112_create_bookmarks.rb      |  17 ++++
 db/schema.rb                                       |  12 +++
 .../api/v1/bookmarks_controller_spec.rb            |  78 ++++++++++++++++
 .../api/v1/statuses/bookmarks_controller_spec.rb   |  57 +++++++++++
 spec/fabricators/bookmark_fabricator.rb            |   4 +
 33 files changed, 712 insertions(+), 1 deletion(-)
 create mode 100644 app/controllers/api/v1/bookmarks_controller.rb
 create mode 100644 app/controllers/api/v1/statuses/bookmarks_controller.rb
 create mode 100644 app/javascript/mastodon/actions/bookmarks.js
 create mode 100644 app/javascript/mastodon/features/bookmarked_statuses/index.js
 create mode 100644 app/models/bookmark.rb
 create mode 100644 db/migrate/20180831171112_create_bookmarks.rb
 create mode 100644 spec/controllers/api/v1/bookmarks_controller_spec.rb
 create mode 100644 spec/controllers/api/v1/statuses/bookmarks_controller_spec.rb
 create mode 100644 spec/fabricators/bookmark_fabricator.rb

(limited to 'app/models')

diff --git a/app/controllers/api/v1/bookmarks_controller.rb b/app/controllers/api/v1/bookmarks_controller.rb
new file mode 100644
index 000000000..cf4cba8dd
--- /dev/null
+++ b/app/controllers/api/v1/bookmarks_controller.rb
@@ -0,0 +1,67 @@
+# frozen_string_literal: true
+
+class Api::V1::BookmarksController < Api::BaseController
+  before_action -> { doorkeeper_authorize! :read, :'read:bookmarks' }
+  before_action :require_user!
+  after_action :insert_pagination_headers
+
+  respond_to :json
+
+  def index
+    @statuses = load_statuses
+    render json: @statuses, each_serializer: REST::StatusSerializer, relationships: StatusRelationshipsPresenter.new(@statuses, current_user&.account_id)
+  end
+
+  private
+
+  def load_statuses
+    cached_bookmarks
+  end
+
+  def cached_bookmarks
+    cache_collection(
+      Status.reorder(nil).joins(:bookmarks).merge(results),
+      Status
+    )
+  end
+
+  def results
+    @_results ||= account_bookmarks.paginate_by_max_id(
+      limit_param(DEFAULT_STATUSES_LIMIT),
+      params[:max_id],
+      params[:since_id]
+    )
+  end
+
+  def account_bookmarks
+    current_account.bookmarks
+  end
+
+  def insert_pagination_headers
+    set_pagination_headers(next_path, prev_path)
+  end
+
+  def next_path
+    api_v1_bookmarks_url pagination_params(max_id: pagination_max_id) if records_continue?
+  end
+
+  def prev_path
+    api_v1_bookmarks_url pagination_params(since_id: pagination_since_id) unless results.empty?
+  end
+
+  def pagination_max_id
+    results.last.id
+  end
+
+  def pagination_since_id
+    results.first.id
+  end
+
+  def records_continue?
+    results.size == limit_param(DEFAULT_STATUSES_LIMIT)
+  end
+
+  def pagination_params(core_params)
+    params.slice(:limit).permit(:limit).merge(core_params)
+  end
+end
diff --git a/app/controllers/api/v1/statuses/bookmarks_controller.rb b/app/controllers/api/v1/statuses/bookmarks_controller.rb
new file mode 100644
index 000000000..bb9729cf5
--- /dev/null
+++ b/app/controllers/api/v1/statuses/bookmarks_controller.rb
@@ -0,0 +1,39 @@
+# frozen_string_literal: true
+
+class Api::V1::Statuses::BookmarksController < Api::BaseController
+  include Authorization
+
+  before_action -> { doorkeeper_authorize! :write, :'write:bookmarks' }
+  before_action :require_user!
+
+  respond_to :json
+
+  def create
+    @status = bookmarked_status
+    render json: @status, serializer: REST::StatusSerializer
+  end
+
+  def destroy
+    @status = requested_status
+    @bookmarks_map = { @status.id => false }
+
+    bookmark = Bookmark.find_by!(account: current_user.account, status: @status)
+    bookmark.destroy!
+
+    render json: @status, serializer: REST::StatusSerializer, relationships: StatusRelationshipsPresenter.new([@status], current_user&.account_id, bookmarks_map: @bookmarks_map)
+  end
+
+  private
+
+  def bookmarked_status
+    authorize_with current_user.account, requested_status, :show?
+
+    bookmark = Bookmark.find_or_create_by!(account: current_user.account, status: requested_status)
+
+    bookmark.status.reload
+  end
+
+  def requested_status
+    Status.find(params[:status_id])
+  end
+end
diff --git a/app/javascript/mastodon/actions/bookmarks.js b/app/javascript/mastodon/actions/bookmarks.js
new file mode 100644
index 000000000..544ed2ff2
--- /dev/null
+++ b/app/javascript/mastodon/actions/bookmarks.js
@@ -0,0 +1,90 @@
+import api, { getLinks } from '../api';
+import { importFetchedStatuses } from './importer';
+
+export const BOOKMARKED_STATUSES_FETCH_REQUEST = 'BOOKMARKED_STATUSES_FETCH_REQUEST';
+export const BOOKMARKED_STATUSES_FETCH_SUCCESS = 'BOOKMARKED_STATUSES_FETCH_SUCCESS';
+export const BOOKMARKED_STATUSES_FETCH_FAIL    = 'BOOKMARKED_STATUSES_FETCH_FAIL';
+
+export const BOOKMARKED_STATUSES_EXPAND_REQUEST = 'BOOKMARKED_STATUSES_EXPAND_REQUEST';
+export const BOOKMARKED_STATUSES_EXPAND_SUCCESS = 'BOOKMARKED_STATUSES_EXPAND_SUCCESS';
+export const BOOKMARKED_STATUSES_EXPAND_FAIL    = 'BOOKMARKED_STATUSES_EXPAND_FAIL';
+
+export function fetchBookmarkedStatuses() {
+  return (dispatch, getState) => {
+    if (getState().getIn(['status_lists', 'bookmarks', 'isLoading'])) {
+      return;
+    }
+
+    dispatch(fetchBookmarkedStatusesRequest());
+
+    api(getState).get('/api/v1/bookmarks').then(response => {
+      const next = getLinks(response).refs.find(link => link.rel === 'next');
+      dispatch(importFetchedStatuses(response.data));
+      dispatch(fetchBookmarkedStatusesSuccess(response.data, next ? next.uri : null));
+    }).catch(error => {
+      dispatch(fetchBookmarkedStatusesFail(error));
+    });
+  };
+};
+
+export function fetchBookmarkedStatusesRequest() {
+  return {
+    type: BOOKMARKED_STATUSES_FETCH_REQUEST,
+  };
+};
+
+export function fetchBookmarkedStatusesSuccess(statuses, next) {
+  return {
+    type: BOOKMARKED_STATUSES_FETCH_SUCCESS,
+    statuses,
+    next,
+  };
+};
+
+export function fetchBookmarkedStatusesFail(error) {
+  return {
+    type: BOOKMARKED_STATUSES_FETCH_FAIL,
+    error,
+  };
+};
+
+export function expandBookmarkedStatuses() {
+  return (dispatch, getState) => {
+    const url = getState().getIn(['status_lists', 'bookmarks', 'next'], null);
+
+    if (url === null || getState().getIn(['status_lists', 'bookmarks', 'isLoading'])) {
+      return;
+    }
+
+    dispatch(expandBookmarkedStatusesRequest());
+
+    api(getState).get(url).then(response => {
+      const next = getLinks(response).refs.find(link => link.rel === 'next');
+      dispatch(importFetchedStatuses(response.data));
+      dispatch(expandBookmarkedStatusesSuccess(response.data, next ? next.uri : null));
+    }).catch(error => {
+      dispatch(expandBookmarkedStatusesFail(error));
+    });
+  };
+};
+
+export function expandBookmarkedStatusesRequest() {
+  return {
+    type: BOOKMARKED_STATUSES_EXPAND_REQUEST,
+  };
+};
+
+export function expandBookmarkedStatusesSuccess(statuses, next) {
+  return {
+    type: BOOKMARKED_STATUSES_EXPAND_SUCCESS,
+    statuses,
+    next,
+  };
+};
+
+export function expandBookmarkedStatusesFail(error) {
+  return {
+    type: BOOKMARKED_STATUSES_EXPAND_FAIL,
+    error,
+  };
+};
diff --git a/app/javascript/mastodon/actions/interactions.js b/app/javascript/mastodon/actions/interactions.js
index 2dc4c574c..28c6b1a62 100644
--- a/app/javascript/mastodon/actions/interactions.js
+++ b/app/javascript/mastodon/actions/interactions.js
@@ -33,6 +33,14 @@ export const UNPIN_REQUEST = 'UNPIN_REQUEST';
 export const UNPIN_SUCCESS = 'UNPIN_SUCCESS';
 export const UNPIN_FAIL    = 'UNPIN_FAIL';
 
+export const BOOKMARK_REQUEST = 'BOOKMARK_REQUEST';
+export const BOOKMARK_SUCCESS = 'BOOKMARKED_SUCCESS';
+export const BOOKMARK_FAIL    = 'BOOKMARKED_FAIL';
+
+export const UNBOOKMARK_REQUEST = 'UNBOOKMARKED_REQUEST';
+export const UNBOOKMARK_SUCCESS = 'UNBOOKMARKED_SUCCESS';
+export const UNBOOKMARK_FAIL    = 'UNBOOKMARKED_FAIL';
+
 export function reblog(status) {
   return function (dispatch, getState) {
     dispatch(reblogRequest(status));
@@ -187,6 +195,78 @@ export function unfavouriteFail(status, error) {
   };
 };
 
+export function bookmark(status) {
+  return function (dispatch, getState) {
+    dispatch(bookmarkRequest(status));
+
+    api(getState).post(`/api/v1/statuses/${status.get('id')}/bookmark`).then(function (response) {
+      dispatch(importFetchedStatus(response.data));
+      dispatch(bookmarkSuccess(status, response.data));
+    }).catch(function (error) {
+      dispatch(bookmarkFail(status, error));
+    });
+  };
+};
+
+export function unbookmark(status) {
+  return (dispatch, getState) => {
+    dispatch(unbookmarkRequest(status));
+
+    api(getState).post(`/api/v1/statuses/${status.get('id')}/unbookmark`).then(response => {
+      dispatch(importFetchedStatus(response.data));
+      dispatch(unbookmarkSuccess(status, response.data));
+    }).catch(error => {
+      dispatch(unbookmarkFail(status, error));
+    });
+  };
+};
+
+export function bookmarkRequest(status) {
+  return {
+    type: BOOKMARK_REQUEST,
+    status: status,
+  };
+};
+
+export function bookmarkSuccess(status, response) {
+  return {
+    type: BOOKMARK_SUCCESS,
+    status: status,
+    response: response,
+  };
+};
+
+export function bookmarkFail(status, error) {
+  return {
+    type: BOOKMARK_FAIL,
+    status: status,
+    error: error,
+  };
+};
+
+export function unbookmarkRequest(status) {
+  return {
+    type: UNBOOKMARK_REQUEST,
+    status: status,
+  };
+};
+
+export function unbookmarkSuccess(status, response) {
+  return {
+    type: UNBOOKMARK_SUCCESS,
+    status: status,
+    response: response,
+  };
+};
+
+export function unbookmarkFail(status, error) {
+  return {
+    type: UNBOOKMARK_FAIL,
+    status: status,
+    error: error,
+  };
+};
+
 export function fetchReblogs(id) {
   return (dispatch, getState) => {
     dispatch(fetchReblogsRequest(id));
diff --git a/app/javascript/mastodon/components/status_action_bar.js b/app/javascript/mastodon/components/status_action_bar.js
index 0bfbd8879..4fa2c1158 100644
--- a/app/javascript/mastodon/components/status_action_bar.js
+++ b/app/javascript/mastodon/components/status_action_bar.js
@@ -23,6 +23,7 @@ const messages = defineMessages({
   cancel_reblog_private: { id: 'status.cancel_reblog_private', defaultMessage: 'Unboost' },
   cannot_reblog: { id: 'status.cannot_reblog', defaultMessage: 'This post cannot be boosted' },
   favourite: { id: 'status.favourite', defaultMessage: 'Favourite' },
+  bookmark: { id: 'status.bookmark', defaultMessage: 'Bookmark' },
   open: { id: 'status.open', defaultMessage: 'Expand this status' },
   report: { id: 'status.report', defaultMessage: 'Report @{name}' },
   muteConversation: { id: 'status.mute_conversation', defaultMessage: 'Mute conversation' },
@@ -66,6 +67,7 @@ class StatusActionBar extends ImmutablePureComponent {
     onEmbed: PropTypes.func,
     onMuteConversation: PropTypes.func,
     onPin: PropTypes.func,
+    onBookmark: PropTypes.func,
     withDismiss: PropTypes.bool,
     intl: PropTypes.object.isRequired,
   };
@@ -114,6 +116,10 @@ class StatusActionBar extends ImmutablePureComponent {
     window.open(`/interact/${this.props.status.get('id')}?type=${type}`, 'mastodon-intent', 'width=445,height=600,resizable=no,menubar=no,status=no,scrollbars=yes');
   }
 
+  handleBookmarkClick = () => {
+    this.props.onBookmark(this.props.status);
+  }
+
   handleDeleteClick = () => {
     this.props.onDelete(this.props.status, this.context.router.history);
   }
@@ -253,6 +259,7 @@ class StatusActionBar extends ImmutablePureComponent {
         <IconButton className='status__action-bar-button' disabled={!publicStatus} active={status.get('reblogged')} pressed={status.get('reblogged')} title={!publicStatus ? intl.formatMessage(messages.cannot_reblog) : intl.formatMessage(messages.reblog)} icon={reblogIcon} onClick={this.handleReblogClick} />
         <IconButton className='status__action-bar-button star-icon' animate active={status.get('favourited')} pressed={status.get('favourited')} title={intl.formatMessage(messages.favourite)} icon='star' onClick={this.handleFavouriteClick} />
         {shareButton}
+        <IconButton className='status__action-bar-button bookmark-icon' disabled={anonymousAccess} active={status.get('bookmarked')} pressed={status.get('bookmarked')} title={intl.formatMessage(messages.bookmark)} icon='bookmark' onClick={this.handleBookmarkClick} />
 
         <div className='status__action-bar-dropdown'>
           <DropdownMenuContainer disabled={anonymousAccess} status={status} items={menu} icon='ellipsis-h' size={18} direction='right' title={intl.formatMessage(messages.more)} />
diff --git a/app/javascript/mastodon/containers/status_container.js b/app/javascript/mastodon/containers/status_container.js
index fb22676e0..16ba02e12 100644
--- a/app/javascript/mastodon/containers/status_container.js
+++ b/app/javascript/mastodon/containers/status_container.js
@@ -9,8 +9,10 @@ import {
 import {
   reblog,
   favourite,
+  bookmark,
   unreblog,
   unfavourite,
+  unbookmark,
   pin,
   unpin,
 } from '../actions/interactions';
@@ -90,6 +92,14 @@ const mapDispatchToProps = (dispatch, { intl }) => ({
     }
   },
 
+  onBookmark (status) {
+    if (status.get('bookmarked')) {
+      dispatch(unbookmark(status));
+    } else {
+      dispatch(bookmark(status));
+    }
+  },
+
   onPin (status) {
     if (status.get('pinned')) {
       dispatch(unpin(status));
diff --git a/app/javascript/mastodon/features/bookmarked_statuses/index.js b/app/javascript/mastodon/features/bookmarked_statuses/index.js
new file mode 100644
index 000000000..c37cb9176
--- /dev/null
+++ b/app/javascript/mastodon/features/bookmarked_statuses/index.js
@@ -0,0 +1,104 @@
+import React from 'react';
+import { connect } from 'react-redux';
+import PropTypes from 'prop-types';
+import ImmutablePropTypes from 'react-immutable-proptypes';
+import { fetchBookmarkedStatuses, expandBookmarkedStatuses } from '../../actions/bookmarks';
+import Column from '../ui/components/column';
+import ColumnHeader from '../../components/column_header';
+import { addColumn, removeColumn, moveColumn } from '../../actions/columns';
+import StatusList from '../../components/status_list';
+import { defineMessages, injectIntl, FormattedMessage } from 'react-intl';
+import ImmutablePureComponent from 'react-immutable-pure-component';
+import { debounce } from 'lodash';
+
+const messages = defineMessages({
+  heading: { id: 'column.bookmarks', defaultMessage: 'Bookmarks' },
+});
+
+const mapStateToProps = state => ({
+  statusIds: state.getIn(['status_lists', 'bookmarks', 'items']),
+  isLoading: state.getIn(['status_lists', 'bookmarks', 'isLoading'], true),
+  hasMore: !!state.getIn(['status_lists', 'bookmarks', 'next']),
+});
+
+export default @connect(mapStateToProps)
+@injectIntl
+class Bookmarks extends ImmutablePureComponent {
+
+  static propTypes = {
+    dispatch: PropTypes.func.isRequired,
+    shouldUpdateScroll: PropTypes.func,
+    statusIds: ImmutablePropTypes.list.isRequired,
+    intl: PropTypes.object.isRequired,
+    columnId: PropTypes.string,
+    multiColumn: PropTypes.bool,
+    hasMore: PropTypes.bool,
+    isLoading: PropTypes.bool,
+  };
+
+  componentWillMount () {
+    this.props.dispatch(fetchBookmarkedStatuses());
+  }
+
+  handlePin = () => {
+    const { columnId, dispatch } = this.props;
+
+    if (columnId) {
+      dispatch(removeColumn(columnId));
+    } else {
+      dispatch(addColumn('BOOKMARKS', {}));
+    }
+  }
+
+  handleMove = (dir) => {
+    const { columnId, dispatch } = this.props;
+    dispatch(moveColumn(columnId, dir));
+  }
+
+  handleHeaderClick = () => {
+    this.column.scrollTop();
+  }
+
+  setRef = c => {
+    this.column = c;
+  }
+
+  handleLoadMore = debounce(() => {
+    this.props.dispatch(expandBookmarkedStatuses());
+  }, 300, { leading: true })
+
+  render () {
+    const { intl, shouldUpdateScroll, statusIds, columnId, multiColumn, hasMore, isLoading } = this.props;
+    const pinned = !!columnId;
+
+    const emptyMessage = <FormattedMessage id='empty_column.bookmarked_statuses' defaultMessage="You don't have any bookmarked toots yet. When you bookmark one, it will show up here." />;
+
+    return (
+      <Column bindToDocument={!multiColumn} ref={this.setRef} label={intl.formatMessage(messages.heading)}>
+        <ColumnHeader
+          icon='bookmark'
+          title={intl.formatMessage(messages.heading)}
+          onPin={this.handlePin}
+          onMove={this.handleMove}
+          onClick={this.handleHeaderClick}
+          pinned={pinned}
+          multiColumn={multiColumn}
+          showBackButton
+        />
+
+        <StatusList
+          trackScroll={!pinned}
+          statusIds={statusIds}
+          scrollKey={`bookmarked_statuses-${columnId}`}
+          hasMore={hasMore}
+          isLoading={isLoading}
+          onLoadMore={this.handleLoadMore}
+          shouldUpdateScroll={shouldUpdateScroll}
+          emptyMessage={emptyMessage}
+          bindToDocument={!multiColumn}
+        />
+      </Column>
+    );
+  }
+
+}
diff --git a/app/javascript/mastodon/features/getting_started/index.js b/app/javascript/mastodon/features/getting_started/index.js
index 67ec7665b..adbc147d1 100644
--- a/app/javascript/mastodon/features/getting_started/index.js
+++ b/app/javascript/mastodon/features/getting_started/index.js
@@ -22,6 +22,7 @@ const messages = defineMessages({
   settings_subheading: { id: 'column_subheading.settings', defaultMessage: 'Settings' },
   community_timeline: { id: 'navigation_bar.community_timeline', defaultMessage: 'Local timeline' },
   direct: { id: 'navigation_bar.direct', defaultMessage: 'Direct messages' },
+  bookmarks: { id: 'navigation_bar.bookmarks', defaultMessage: 'Bookmarks' },
   preferences: { id: 'navigation_bar.preferences', defaultMessage: 'Preferences' },
   follow_requests: { id: 'navigation_bar.follow_requests', defaultMessage: 'Follow requests' },
   favourites: { id: 'navigation_bar.favourites', defaultMessage: 'Favourites' },
@@ -126,11 +127,12 @@ class GettingStarted extends ImmutablePureComponent {
 
     navItems.push(
       <ColumnLink key={i++} icon='envelope' text={intl.formatMessage(messages.direct)} to='/timelines/direct' />,
+      <ColumnLink key={i++} icon='bookmark' text={intl.formatMessage(messages.bookmarks)} to='/bookmarks' />,
       <ColumnLink key={i++} icon='star' text={intl.formatMessage(messages.favourites)} to='/favourites' />,
       <ColumnLink key={i++} icon='list-ul' text={intl.formatMessage(messages.lists)} to='/lists' />
     );
 
-    height += 48*3;
+    height += 48*4;
 
     if (myAccount.get('locked') || unreadFollowRequests > 0) {
       navItems.push(<ColumnLink key={i++} icon='user-plus' text={intl.formatMessage(messages.follow_requests)} badge={badgeDisplay(unreadFollowRequests, 40)} to='/follow_requests' />);
diff --git a/app/javascript/mastodon/features/status/components/action_bar.js b/app/javascript/mastodon/features/status/components/action_bar.js
index 3e511b7a6..1b81cd245 100644
--- a/app/javascript/mastodon/features/status/components/action_bar.js
+++ b/app/javascript/mastodon/features/status/components/action_bar.js
@@ -17,6 +17,7 @@ const messages = defineMessages({
   cancel_reblog_private: { id: 'status.cancel_reblog_private', defaultMessage: 'Unboost' },
   cannot_reblog: { id: 'status.cannot_reblog', defaultMessage: 'This post cannot be boosted' },
   favourite: { id: 'status.favourite', defaultMessage: 'Favourite' },
+  bookmark: { id: 'status.bookmark', defaultMessage: 'Bookmark' },
   mute: { id: 'status.mute', defaultMessage: 'Mute @{name}' },
   muteConversation: { id: 'status.mute_conversation', defaultMessage: 'Mute conversation' },
   unmuteConversation: { id: 'status.unmute_conversation', defaultMessage: 'Unmute conversation' },
@@ -43,6 +44,7 @@ class ActionBar extends React.PureComponent {
     onReply: PropTypes.func.isRequired,
     onReblog: PropTypes.func.isRequired,
     onFavourite: PropTypes.func.isRequired,
+    onBookmark: PropTypes.func.isRequired,
     onDelete: PropTypes.func.isRequired,
     onDirect: PropTypes.func.isRequired,
     onMention: PropTypes.func.isRequired,
@@ -67,6 +69,10 @@ class ActionBar extends React.PureComponent {
     this.props.onFavourite(this.props.status);
   }
 
+  handleBookmarkClick = (e) => {
+    this.props.onBookmark(this.props.status, e);
+  }
+
   handleDeleteClick = () => {
     this.props.onDelete(this.props.status, this.context.router.history);
   }
@@ -198,6 +204,7 @@ class ActionBar extends React.PureComponent {
         <div className='detailed-status__button'><IconButton disabled={reblog_disabled} active={status.get('reblogged')} title={reblog_disabled ? intl.formatMessage(messages.cannot_reblog) : intl.formatMessage(messages.reblog)} icon={reblogIcon} onClick={this.handleReblogClick} /></div>
         <div className='detailed-status__button'><IconButton className='star-icon' animate active={status.get('favourited')} title={intl.formatMessage(messages.favourite)} icon='star' onClick={this.handleFavouriteClick} /></div>
         {shareButton}
+        <div className='detailed-status__button'><IconButton className='bookmark-icon' active={status.get('bookmarked')} title={intl.formatMessage(messages.bookmark)} icon='bookmark' onClick={this.handleBookmarkClick} /></div>
 
         <div className='detailed-status__action-bar-dropdown'>
           <DropdownMenuContainer size={18} icon='ellipsis-h' items={menu} direction='left' title='More' />
diff --git a/app/javascript/mastodon/features/status/index.js b/app/javascript/mastodon/features/status/index.js
index 029057d40..9fb3fe305 100644
--- a/app/javascript/mastodon/features/status/index.js
+++ b/app/javascript/mastodon/features/status/index.js
@@ -13,6 +13,8 @@ import Column from '../ui/components/column';
 import {
   favourite,
   unfavourite,
+  bookmark,
+  unbookmark,
   reblog,
   unreblog,
   pin,
@@ -232,6 +234,14 @@ class Status extends ImmutablePureComponent {
     }
   }
 
+  handleBookmarkClick = (status) => {
+    if (status.get('bookmarked')) {
+      this.props.dispatch(unbookmark(status));
+    } else {
+      this.props.dispatch(bookmark(status));
+    }
+  }
+
   handleDeleteClick = (status, history, withRedraft = false) => {
     const { dispatch, intl } = this.props;
 
@@ -499,6 +509,7 @@ class Status extends ImmutablePureComponent {
                   onReply={this.handleReplyClick}
                   onFavourite={this.handleFavouriteClick}
                   onReblog={this.handleReblogClick}
+                  onBookmark={this.handleBookmarkClick}
                   onDelete={this.handleDeleteClick}
                   onDirect={this.handleDirectClick}
                   onMention={this.handleMentionClick}
diff --git a/app/javascript/mastodon/features/ui/components/columns_area.js b/app/javascript/mastodon/features/ui/components/columns_area.js
index f31425c70..8bc4bfc0e 100644
--- a/app/javascript/mastodon/features/ui/components/columns_area.js
+++ b/app/javascript/mastodon/features/ui/components/columns_area.js
@@ -21,6 +21,7 @@ import {
   HashtagTimeline,
   DirectTimeline,
   FavouritedStatuses,
+  BookmarkedStatuses,
   ListTimeline,
   Directory,
 } from '../../ui/util/async-components';
@@ -40,6 +41,7 @@ const componentMap = {
   'HASHTAG': HashtagTimeline,
   'DIRECT': DirectTimeline,
   'FAVOURITES': FavouritedStatuses,
+  'BOOKMARKS': BookmarkedStatuses,
   'LIST': ListTimeline,
   'DIRECTORY': Directory,
 };
diff --git a/app/javascript/mastodon/features/ui/components/navigation_panel.js b/app/javascript/mastodon/features/ui/components/navigation_panel.js
index 51e3ec037..0c12852f5 100644
--- a/app/javascript/mastodon/features/ui/components/navigation_panel.js
+++ b/app/javascript/mastodon/features/ui/components/navigation_panel.js
@@ -17,6 +17,7 @@ const NavigationPanel = () => (
     <NavLink className='column-link column-link--transparent' exact to='/timelines/public' data-preview-title-id='column.public' data-preview-icon='globe' ><Icon className='column-link__icon' id='globe' fixedWidth /><FormattedMessage id='tabs_bar.federated_timeline' defaultMessage='Federated' /></NavLink>
     <NavLink className='column-link column-link--transparent' to='/timelines/direct'><Icon className='column-link__icon' id='envelope' fixedWidth /><FormattedMessage id='navigation_bar.direct' defaultMessage='Direct messages' /></NavLink>
     <NavLink className='column-link column-link--transparent' to='/favourites'><Icon className='column-link__icon' id='star' fixedWidth /><FormattedMessage id='navigation_bar.favourites' defaultMessage='Favourites' /></NavLink>
+    <NavLink className='column-link column-link--transparent' to='/bookmarks'><Icon className='column-link__icon' id='bookmark' fixedWidth /><FormattedMessage id='navigation_bar.bookmarks' defaultMessage='Bookmarks' /></NavLink>
     <NavLink className='column-link column-link--transparent' to='/lists'><Icon className='column-link__icon' id='list-ul' fixedWidth /><FormattedMessage id='navigation_bar.lists' defaultMessage='Lists' /></NavLink>
     {profile_directory && <NavLink className='column-link column-link--transparent' to='/directory'><Icon className='column-link__icon' id='address-book-o' fixedWidth /><FormattedMessage id='getting_started.directory' defaultMessage='Profile directory' /></NavLink>}
 
diff --git a/app/javascript/mastodon/features/ui/index.js b/app/javascript/mastodon/features/ui/index.js
index a45ba91eb..b0e38c5cb 100644
--- a/app/javascript/mastodon/features/ui/index.js
+++ b/app/javascript/mastodon/features/ui/index.js
@@ -41,6 +41,7 @@ import {
   FollowRequests,
   GenericNotFound,
   FavouritedStatuses,
+  BookmarkedStatuses,
   ListTimeline,
   Blocks,
   DomainBlocks,
@@ -190,6 +191,7 @@ class SwitchingColumnsArea extends React.PureComponent {
 
           <WrappedRoute path='/notifications' component={Notifications} content={children} componentParams={{ shouldUpdateScroll: this.shouldUpdateScroll }} />
           <WrappedRoute path='/favourites' component={FavouritedStatuses} content={children} componentParams={{ shouldUpdateScroll: this.shouldUpdateScroll }} />
+          <WrappedRoute path='/bookmarks' component={BookmarkedStatuses} content={children} />
           <WrappedRoute path='/pinned' component={PinnedStatuses} content={children} componentParams={{ shouldUpdateScroll: this.shouldUpdateScroll }} />
 
           <WrappedRoute path='/search' component={Search} content={children} />
diff --git a/app/javascript/mastodon/features/ui/util/async-components.js b/app/javascript/mastodon/features/ui/util/async-components.js
index bb0fcb859..986efda1e 100644
--- a/app/javascript/mastodon/features/ui/util/async-components.js
+++ b/app/javascript/mastodon/features/ui/util/async-components.js
@@ -90,6 +90,10 @@ export function FavouritedStatuses () {
   return import(/* webpackChunkName: "features/favourited_statuses" */'../../favourited_statuses');
 }
 
+export function BookmarkedStatuses () {
+  return import(/* webpackChunkName: "features/bookmarked_statuses" */'../../bookmarked_statuses');
+}
+
 export function Blocks () {
   return import(/* webpackChunkName: "features/blocks" */'../../blocks');
 }
diff --git a/app/javascript/mastodon/reducers/status_lists.js b/app/javascript/mastodon/reducers/status_lists.js
index 6c5f33557..9f8f28dee 100644
--- a/app/javascript/mastodon/reducers/status_lists.js
+++ b/app/javascript/mastodon/reducers/status_lists.js
@@ -6,6 +6,14 @@ import {
   FAVOURITED_STATUSES_EXPAND_SUCCESS,
   FAVOURITED_STATUSES_EXPAND_FAIL,
 } from '../actions/favourites';
+import {
+  BOOKMARKED_STATUSES_FETCH_REQUEST,
+  BOOKMARKED_STATUSES_FETCH_SUCCESS,
+  BOOKMARKED_STATUSES_FETCH_FAIL,
+  BOOKMARKED_STATUSES_EXPAND_REQUEST,
+  BOOKMARKED_STATUSES_EXPAND_SUCCESS,
+  BOOKMARKED_STATUSES_EXPAND_FAIL,
+} from '../actions/bookmarks';
 import {
   PINNED_STATUSES_FETCH_SUCCESS,
 } from '../actions/pin_statuses';
@@ -13,6 +21,8 @@ import { Map as ImmutableMap, List as ImmutableList } from 'immutable';
 import {
   FAVOURITE_SUCCESS,
   UNFAVOURITE_SUCCESS,
+  BOOKMARK_SUCCESS,
+  UNBOOKMARK_SUCCESS,
   PIN_SUCCESS,
   UNPIN_SUCCESS,
 } from '../actions/interactions';
@@ -23,6 +33,11 @@ const initialState = ImmutableMap({
     loaded: false,
     items: ImmutableList(),
   }),
+  bookmarks: ImmutableMap({
+    next: null,
+    loaded: false,
+    items: ImmutableList(),
+  }),
   pins: ImmutableMap({
     next: null,
     loaded: false,
@@ -71,10 +86,24 @@ export default function statusLists(state = initialState, action) {
     return normalizeList(state, 'favourites', action.statuses, action.next);
   case FAVOURITED_STATUSES_EXPAND_SUCCESS:
     return appendToList(state, 'favourites', action.statuses, action.next);
+  case BOOKMARKED_STATUSES_FETCH_REQUEST:
+  case BOOKMARKED_STATUSES_EXPAND_REQUEST:
+    return state.setIn(['bookmarks', 'isLoading'], true);
+  case BOOKMARKED_STATUSES_FETCH_FAIL:
+  case BOOKMARKED_STATUSES_EXPAND_FAIL:
+    return state.setIn(['bookmarks', 'isLoading'], false);
+  case BOOKMARKED_STATUSES_FETCH_SUCCESS:
+    return normalizeList(state, 'bookmarks', action.statuses, action.next);
+  case BOOKMARKED_STATUSES_EXPAND_SUCCESS:
+    return appendToList(state, 'bookmarks', action.statuses, action.next);
   case FAVOURITE_SUCCESS:
     return prependOneToList(state, 'favourites', action.status);
   case UNFAVOURITE_SUCCESS:
     return removeOneFromList(state, 'favourites', action.status);
+  case BOOKMARK_SUCCESS:
+    return prependOneToList(state, 'bookmarks', action.status);
+  case UNBOOKMARK_SUCCESS:
+    return removeOneFromList(state, 'bookmarks', action.status);
   case PINNED_STATUSES_FETCH_SUCCESS:
     return normalizeList(state, 'pins', action.statuses, action.next);
   case PIN_SUCCESS:
diff --git a/app/javascript/mastodon/reducers/statuses.js b/app/javascript/mastodon/reducers/statuses.js
index 372673bc0..772f98bcb 100644
--- a/app/javascript/mastodon/reducers/statuses.js
+++ b/app/javascript/mastodon/reducers/statuses.js
@@ -4,6 +4,8 @@ import {
   FAVOURITE_REQUEST,
   FAVOURITE_FAIL,
   UNFAVOURITE_SUCCESS,
+  BOOKMARK_REQUEST,
+  BOOKMARK_FAIL,
 } from '../actions/interactions';
 import {
   STATUS_MUTE_SUCCESS,
@@ -43,6 +45,10 @@ export default function statuses(state = initialState, action) {
     return state.setIn([action.status.get('id'), 'favourites_count'], favouritesCount - 1);
   case FAVOURITE_FAIL:
     return state.get(action.status.get('id')) === undefined ? state : state.setIn([action.status.get('id'), 'favourited'], false);
+  case BOOKMARK_REQUEST:
+    return state.get(action.status.get('id')) === undefined ? state : state.setIn([action.status.get('id'), 'bookmarked'], true);
+  case BOOKMARK_FAIL:
+    return state.get(action.status.get('id')) === undefined ? state : state.setIn([action.status.get('id'), 'bookmarked'], false);
   case REBLOG_REQUEST:
     return state.setIn([action.status.get('id'), 'reblogged'], true);
   case REBLOG_FAIL:
diff --git a/app/javascript/styles/mastodon/components.scss b/app/javascript/styles/mastodon/components.scss
index ded668aea..98ffe1c20 100644
--- a/app/javascript/styles/mastodon/components.scss
+++ b/app/javascript/styles/mastodon/components.scss
@@ -1592,6 +1592,10 @@ a.account__display-name {
   color: $gold-star;
 }
 
+.bookmark-icon.active {
+  color: $red-bookmark;
+}
+
 .no-reduce-motion .icon-button.star-icon {
   &.activate {
     & > .fa-star {
diff --git a/app/javascript/styles/mastodon/variables.scss b/app/javascript/styles/mastodon/variables.scss
index a82c44229..8602c3dde 100644
--- a/app/javascript/styles/mastodon/variables.scss
+++ b/app/javascript/styles/mastodon/variables.scss
@@ -6,6 +6,8 @@ $error-red: #df405a !default;        // Cerise
 $warning-red: #ff5050 !default;      // Sunset Orange
 $gold-star: #ca8f04 !default;        // Dark Goldenrod
 
+$red-bookmark: $warning-red;
+
 // Values from the classic Mastodon UI
 $classic-base-color: #282c37;         // Midnight Express
 $classic-primary-color: #9baec8;      // Echo Blue
diff --git a/app/models/bookmark.rb b/app/models/bookmark.rb
new file mode 100644
index 000000000..01dc48ee7
--- /dev/null
+++ b/app/models/bookmark.rb
@@ -0,0 +1,26 @@
+# frozen_string_literal: true
+# == Schema Information
+#
+# Table name: bookmarks
+#
+#  id         :integer          not null, primary key
+#  created_at :datetime         not null
+#  updated_at :datetime         not null
+#  account_id :integer          not null
+#  status_id  :integer          not null
+#
+
+class Bookmark < ApplicationRecord
+  include Paginable
+
+  update_index('statuses#status', :status) if Chewy.enabled?
+
+  belongs_to :account, inverse_of: :bookmarks
+  belongs_to :status,  inverse_of: :bookmarks
+
+  validates :status_id, uniqueness: { scope: :account_id }
+
+  before_validation do
+    self.status = status.reblog if status&.reblog?
+  end
+end
diff --git a/app/models/concerns/account_associations.rb b/app/models/concerns/account_associations.rb
index c9cc5c610..499edbf4e 100644
--- a/app/models/concerns/account_associations.rb
+++ b/app/models/concerns/account_associations.rb
@@ -13,6 +13,7 @@ module AccountAssociations
     # Timelines
     has_many :statuses, inverse_of: :account, dependent: :destroy
     has_many :favourites, inverse_of: :account, dependent: :destroy
+    has_many :bookmarks, inverse_of: :account, dependent: :destroy
     has_many :mentions, inverse_of: :account, dependent: :destroy
     has_many :notifications, inverse_of: :account, dependent: :destroy
     has_many :conversations, class_name: 'AccountConversation', dependent: :destroy, inverse_of: :account
diff --git a/app/models/concerns/account_interactions.rb b/app/models/concerns/account_interactions.rb
index ad2909d91..f27d39483 100644
--- a/app/models/concerns/account_interactions.rb
+++ b/app/models/concerns/account_interactions.rb
@@ -186,6 +186,10 @@ module AccountInteractions
     status.proper.favourites.where(account: self).exists?
   end
 
+  def bookmarked?(status)
+    status.proper.bookmarks.where(account: self).exists?
+  end
+
   def reblogged?(status)
     status.proper.reblogs.where(account: self).exists?
   end
diff --git a/app/models/status.rb b/app/models/status.rb
index 0c01a5389..1cb381400 100644
--- a/app/models/status.rb
+++ b/app/models/status.rb
@@ -54,6 +54,7 @@ class Status < ApplicationRecord
   belongs_to :reblog, foreign_key: 'reblog_of_id', class_name: 'Status', inverse_of: :reblogs, optional: true
 
   has_many :favourites, inverse_of: :status, dependent: :destroy
+  has_many :bookmarks, inverse_of: :status, dependent: :destroy
   has_many :reblogs, foreign_key: 'reblog_of_id', class_name: 'Status', inverse_of: :reblog, dependent: :destroy
   has_many :replies, foreign_key: 'in_reply_to_id', class_name: 'Status', inverse_of: :thread
   has_many :mentions, dependent: :destroy, inverse_of: :status
@@ -302,6 +303,10 @@ class Status < ApplicationRecord
       Favourite.select('status_id').where(status_id: status_ids).where(account_id: account_id).each_with_object({}) { |f, h| h[f.status_id] = true }
     end
 
+    def bookmarks_map(status_ids, account_id)
+      Bookmark.select('status_id').where(status_id: status_ids).where(account_id: account_id).map { |f| [f.status_id, true] }.to_h
+    end
+
     def reblogs_map(status_ids, account_id)
       unscoped.select('reblog_of_id').where(reblog_of_id: status_ids).where(account_id: account_id).each_with_object({}) { |s, h| h[s.reblog_of_id] = true }
     end
diff --git a/app/presenters/status_relationships_presenter.rb b/app/presenters/status_relationships_presenter.rb
index b04e10e2f..64e688d87 100644
--- a/app/presenters/status_relationships_presenter.rb
+++ b/app/presenters/status_relationships_presenter.rb
@@ -7,6 +7,7 @@ class StatusRelationshipsPresenter
     if current_account_id.nil?
       @reblogs_map    = {}
       @favourites_map = {}
+      @bookmarks_map  = {}
       @mutes_map      = {}
       @pins_map       = {}
     else
@@ -17,6 +18,7 @@ class StatusRelationshipsPresenter
 
       @reblogs_map     = Status.reblogs_map(status_ids, current_account_id).merge(options[:reblogs_map] || {})
       @favourites_map  = Status.favourites_map(status_ids, current_account_id).merge(options[:favourites_map] || {})
+      @bookmarks_map   = Status.bookmarks_map(status_ids, current_account_id).merge(options[:bookmarks_map] || {})
       @mutes_map       = Status.mutes_map(conversation_ids, current_account_id).merge(options[:mutes_map] || {})
       @pins_map        = Status.pins_map(pinnable_status_ids, current_account_id).merge(options[:pins_map] || {})
     end
diff --git a/app/serializers/rest/status_serializer.rb b/app/serializers/rest/status_serializer.rb
index 2dc4a1b61..08bc4d82a 100644
--- a/app/serializers/rest/status_serializer.rb
+++ b/app/serializers/rest/status_serializer.rb
@@ -9,6 +9,7 @@ class REST::StatusSerializer < ActiveModel::Serializer
   attribute :favourited, if: :current_user?
   attribute :reblogged, if: :current_user?
   attribute :muted, if: :current_user?
+  attribute :bookmarked, if: :current_user?
   attribute :pinned, if: :pinnable?
 
   attribute :content, unless: :source_requested?
@@ -93,6 +94,14 @@ class REST::StatusSerializer < ActiveModel::Serializer
     end
   end
 
+  def bookmarked
+    if instance_options && instance_options[:bookmarks]
+      instance_options[:bookmarks].bookmarks_map[object.id] || false
+    else
+      current_user.account.bookmarked?(object)
+    end
+  end
+
   def pinned
     if instance_options && instance_options[:relationships]
       instance_options[:relationships].pins_map[object.id] || false
diff --git a/app/services/backup_service.rb b/app/services/backup_service.rb
index 12e4fa8b4..fe26d7aa0 100644
--- a/app/services/backup_service.rb
+++ b/app/services/backup_service.rb
@@ -45,6 +45,7 @@ class BackupService < BaseService
           dump_media_attachments!(tar)
           dump_outbox!(tar)
           dump_likes!(tar)
+          dump_bookmarks!(tar)
           dump_actor!(tar)
         end
       end
@@ -85,6 +86,7 @@ class BackupService < BaseService
     actor[:image][:url] = 'header' + File.extname(actor[:image][:url]) if actor[:image]
     actor[:outbox]      = 'outbox.json'
     actor[:likes]       = 'likes.json'
+    actor[:bookmarks]   = 'bookmarks.json'
 
     download_to_tar(tar, account.avatar, 'avatar' + File.extname(account.avatar.path)) if account.avatar.exists?
     download_to_tar(tar, account.header, 'header' + File.extname(account.header.path)) if account.header.exists?
@@ -115,6 +117,25 @@ class BackupService < BaseService
     end
   end
 
+  def dump_bookmarks!(tar)
+    collection = serialize(ActivityPub::CollectionPresenter.new(id: 'bookmarks.json', type: :ordered, size: 0, items: []), ActivityPub::CollectionSerializer)
+
+    Status.reorder(nil).joins(:bookmarks).includes(:account).merge(account.bookmarks).find_in_batches do |statuses|
+      statuses.each do |status|
+        collection[:totalItems] += 1
+        collection[:orderedItems] << ActivityPub::TagManager.instance.uri_for(status)
+      end
+
+      GC.start
+    end
+
+    json = Oj.dump(collection)
+
+    tar.add_file_simple('bookmarks.json', 0o444, json.bytesize) do |io|
+      io.write(json)
+    end
+  end
+
   def collection_presenter
     ActivityPub::CollectionPresenter.new(
       id: 'outbox.json',
diff --git a/config/initializers/doorkeeper.rb b/config/initializers/doorkeeper.rb
index 914b3c001..a5c9caa4a 100644
--- a/config/initializers/doorkeeper.rb
+++ b/config/initializers/doorkeeper.rb
@@ -58,6 +58,7 @@ Doorkeeper.configure do
   optional_scopes :write,
                   :'write:accounts',
                   :'write:blocks',
+                  :'write:bookmarks',
                   :'write:conversations',
                   :'write:favourites',
                   :'write:filters',
@@ -71,6 +72,7 @@ Doorkeeper.configure do
                   :read,
                   :'read:accounts',
                   :'read:blocks',
+                  :'read:bookmarks',
                   :'read:favourites',
                   :'read:filters',
                   :'read:follows',
diff --git a/config/locales/doorkeeper.en.yml b/config/locales/doorkeeper.en.yml
index d9b7c2c8e..4e9c83a8f 100644
--- a/config/locales/doorkeeper.en.yml
+++ b/config/locales/doorkeeper.en.yml
@@ -125,6 +125,7 @@ en:
       read: read all your account's data
       read:accounts: see accounts information
       read:blocks: see your blocks
+      read:bookmarks: see your bookmarks
       read:favourites: see your favourites
       read:filters: see your filters
       read:follows: see your follows
@@ -137,6 +138,7 @@ en:
       write: modify all your account's data
       write:accounts: modify your profile
       write:blocks: block accounts and domains
+      write:bookmarks: bookmark statuses
       write:favourites: favourite statuses
       write:filters: create filters
       write:follows: follow people
diff --git a/config/routes.rb b/config/routes.rb
index e43e201a5..5411cff58 100644
--- a/config/routes.rb
+++ b/config/routes.rb
@@ -289,6 +289,9 @@ Rails.application.routes.draw do
           resource :favourite, only: :create
           post :unfavourite, to: 'favourites#destroy'
 
+          resource :bookmark, only: :create
+          post :unbookmark, to: 'bookmarks#destroy'
+
           resource :mute, only: :create
           post :unmute, to: 'mutes#destroy'
 
@@ -324,6 +327,7 @@ Rails.application.routes.draw do
       resources :blocks,       only: [:index]
       resources :mutes,        only: [:index]
       resources :favourites,   only: [:index]
+      resources :bookmarks,    only: [:index]
       resources :reports,      only: [:create]
       resources :trends,       only: [:index]
       resources :filters,      only: [:index, :create, :show, :update, :destroy]
diff --git a/db/migrate/20180831171112_create_bookmarks.rb b/db/migrate/20180831171112_create_bookmarks.rb
new file mode 100644
index 000000000..27c7339c9
--- /dev/null
+++ b/db/migrate/20180831171112_create_bookmarks.rb
@@ -0,0 +1,17 @@
+class CreateBookmarks < ActiveRecord::Migration[5.1]
+  def change
+    create_table :bookmarks do |t|
+      t.references :account, null: false
+      t.references :status, null: false
+
+      t.timestamps
+    end
+
+    safety_assured do
+      add_foreign_key :bookmarks, :accounts, column: :account_id, on_delete: :cascade
+      add_foreign_key :bookmarks, :statuses, column: :status_id, on_delete: :cascade
+    end
+
+    add_index :bookmarks, [:account_id, :status_id], unique: true
+  end
+end
diff --git a/db/schema.rb b/db/schema.rb
index 1bcc003a5..84c76e4ce 100644
--- a/db/schema.rb
+++ b/db/schema.rb
@@ -217,6 +217,16 @@ ActiveRecord::Schema.define(version: 2019_10_31_163205) do
     t.index ["target_account_id"], name: "index_blocks_on_target_account_id"
   end
 
+  create_table "bookmarks", force: :cascade do |t|
+    t.bigint "account_id", null: false
+    t.bigint "status_id", null: false
+    t.datetime "created_at", null: false
+    t.datetime "updated_at", null: false
+    t.index ["account_id", "status_id"], name: "index_bookmarks_on_account_id_and_status_id", unique: true
+    t.index ["account_id"], name: "index_bookmarks_on_account_id"
+    t.index ["status_id"], name: "index_bookmarks_on_status_id"
+  end
+
   create_table "conversation_mutes", force: :cascade do |t|
     t.bigint "conversation_id", null: false
     t.bigint "account_id", null: false
@@ -834,6 +844,8 @@ ActiveRecord::Schema.define(version: 2019_10_31_163205) do
   add_foreign_key "backups", "users", on_delete: :nullify
   add_foreign_key "blocks", "accounts", column: "target_account_id", name: "fk_9571bfabc1", on_delete: :cascade
   add_foreign_key "blocks", "accounts", name: "fk_4269e03e65", on_delete: :cascade
+  add_foreign_key "bookmarks", "accounts", on_delete: :cascade
+  add_foreign_key "bookmarks", "statuses", on_delete: :cascade
   add_foreign_key "conversation_mutes", "accounts", name: "fk_225b4212bb", on_delete: :cascade
   add_foreign_key "conversation_mutes", "conversations", on_delete: :cascade
   add_foreign_key "custom_filters", "accounts", on_delete: :cascade
diff --git a/spec/controllers/api/v1/bookmarks_controller_spec.rb b/spec/controllers/api/v1/bookmarks_controller_spec.rb
new file mode 100644
index 000000000..79601b6e6
--- /dev/null
+++ b/spec/controllers/api/v1/bookmarks_controller_spec.rb
@@ -0,0 +1,78 @@
+require 'rails_helper'
+
+RSpec.describe Api::V1::BookmarksController, type: :controller do
+  render_views
+
+  let(:user)  { Fabricate(:user) }
+  let(:token) { Fabricate(:accessible_access_token, resource_owner_id: user.id, scopes: 'read:bookmarks') }
+
+  describe 'GET #index' do
+    context 'without token' do
+      it 'returns http unauthorized' do
+        get :index
+        expect(response).to have_http_status :unauthorized
+      end
+    end
+
+    context 'with token' do
+      context 'without read scope' do
+        before do
+          allow(controller).to receive(:doorkeeper_token) do
+            Fabricate(:accessible_access_token, resource_owner_id: user.id, scopes: '')
+          end
+        end
+
+        it 'returns http forbidden' do
+          get :index
+          expect(response).to have_http_status :forbidden
+        end
+      end
+
+      context 'without valid resource owner' do
+        before do
+          token = Fabricate(:accessible_access_token, resource_owner_id: user.id, scopes: 'read')
+          user.destroy!
+
+          allow(controller).to receive(:doorkeeper_token) { token }
+        end
+
+        it 'returns http unprocessable entity' do
+          get :index
+          expect(response).to have_http_status :unprocessable_entity
+        end
+      end
+
+      context 'with read scope and valid resource owner' do
+        before do
+          allow(controller).to receive(:doorkeeper_token) do
+            Fabricate(:accessible_access_token, resource_owner_id: user.id, scopes: 'read')
+          end
+        end
+
+        it 'shows bookmarks owned by the user' do
+          bookmarked_by_user = Fabricate(:bookmark, account: user.account)
+          bookmarked_by_others = Fabricate(:bookmark)
+
+          get :index
+
+          expect(assigns(:statuses)).to match_array [bookmarked_by_user.status]
+        end
+
+        it 'adds pagination headers if necessary' do
+          bookmark = Fabricate(:bookmark, account: user.account)
+
+          get :index, params: { limit: 1 }
+
+          expect(response.headers['Link'].find_link(['rel', 'next']).href).to eq "http://test.host/api/v1/bookmarks?limit=1&max_id=#{bookmark.id}"
+          expect(response.headers['Link'].find_link(['rel', 'prev']).href).to eq "http://test.host/api/v1/bookmarks?limit=1&since_id=#{bookmark.id}"
+        end
+
+        it 'does not add pagination headers if not necessary' do
+          get :index
+
+          expect(response.headers['Link']).to eq nil
+        end
+      end
+    end
+  end
+end
diff --git a/spec/controllers/api/v1/statuses/bookmarks_controller_spec.rb b/spec/controllers/api/v1/statuses/bookmarks_controller_spec.rb
new file mode 100644
index 000000000..b79853718
--- /dev/null
+++ b/spec/controllers/api/v1/statuses/bookmarks_controller_spec.rb
@@ -0,0 +1,57 @@
+# frozen_string_literal: true
+
+require 'rails_helper'
+
+describe Api::V1::Statuses::BookmarksController do
+  render_views
+
+  let(:user)  { Fabricate(:user, account: Fabricate(:account, username: 'alice')) }
+  let(:app)   { Fabricate(:application, name: 'Test app', website: 'http://testapp.com') }
+  let(:token) { Fabricate(:accessible_access_token, resource_owner_id: user.id, scopes: 'write:bookmarks', application: app) }
+
+  context 'with an oauth token' do
+    before do
+      allow(controller).to receive(:doorkeeper_token) { token }
+    end
+
+    describe 'POST #create' do
+      let(:status) { Fabricate(:status, account: user.account) }
+
+      before do
+        post :create, params: { status_id: status.id }
+      end
+
+      it 'returns http success' do
+        expect(response).to have_http_status(:success)
+      end
+
+      it 'updates the bookmarked attribute' do
+        expect(user.account.bookmarked?(status)).to be true
+      end
+
+      it 'return json with updated attributes' do
+        hash_body = body_as_json
+
+        expect(hash_body[:id]).to eq status.id.to_s
+        expect(hash_body[:bookmarked]).to be true
+      end
+    end
+
+    describe 'POST #destroy' do
+      let(:status) { Fabricate(:status, account: user.account) }
+
+      before do
+        Bookmark.find_or_create_by!(account: user.account, status: status)
+        post :destroy, params: { status_id: status.id }
+      end
+
+      it 'returns http success' do
+        expect(response).to have_http_status(:success)
+      end
+
+      it 'updates the bookmarked attribute' do
+        expect(user.account.bookmarked?(status)).to be false
+      end
+    end
+  end
+end
diff --git a/spec/fabricators/bookmark_fabricator.rb b/spec/fabricators/bookmark_fabricator.rb
new file mode 100644
index 000000000..12cbc5bfa
--- /dev/null
+++ b/spec/fabricators/bookmark_fabricator.rb
@@ -0,0 +1,4 @@
+Fabricator(:bookmark) do
+  account
+  status
+end
-- 
cgit 


From 2edf6d81cba25555814046d7b9f260872c2c9545 Mon Sep 17 00:00:00 2001
From: ThibG <thib@sitedethib.com>
Date: Thu, 21 Nov 2019 17:35:04 +0100
Subject: Simplify SQL query when performing account search amongst followings
 (#12302)

---
 app/models/account.rb | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

(limited to 'app/models')

diff --git a/app/models/account.rb b/app/models/account.rb
index cae82da16..d17782f78 100644
--- a/app/models/account.rb
+++ b/app/models/account.rb
@@ -436,7 +436,7 @@ class Account < ApplicationRecord
             accounts.*,
             (count(f.id) + 1) * ts_rank_cd(#{textsearch}, #{query}, 32) AS rank
           FROM accounts
-          LEFT OUTER JOIN follows AS f ON (accounts.id = f.account_id AND f.target_account_id = ?) OR (accounts.id = f.target_account_id AND f.account_id = ?)
+          LEFT OUTER JOIN follows AS f ON (accounts.id = f.account_id AND f.target_account_id = ?)
           WHERE accounts.id IN (SELECT * FROM first_degree)
             AND #{query} @@ #{textsearch}
             AND accounts.suspended_at IS NULL
@@ -446,7 +446,7 @@ class Account < ApplicationRecord
           LIMIT ? OFFSET ?
         SQL
 
-        records = find_by_sql([sql, account.id, account.id, account.id, account.id, limit, offset])
+        records = find_by_sql([sql, account.id, account.id, account.id, limit, offset])
       else
         sql = <<-SQL.squish
           SELECT
-- 
cgit 


From b25665d2e87548b79b2bab3eab5c78b2328e2dbb Mon Sep 17 00:00:00 2001
From: Thibaut Girka <thib@sitedethib.com>
Date: Thu, 31 Oct 2019 14:40:58 +0100
Subject: Do not consider moved accounts as non-functional

This might be somewhat controversial, but putting a redirection up
makes the account unusable, which works against use cases such as
using the moved account as backup in case the new one temporarily fails
(sure, the redirection can be temporarily removed, but it is a hassle)
---
 app/models/user.rb | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

(limited to 'app/models')

diff --git a/app/models/user.rb b/app/models/user.rb
index e33610d54..49cfc25ca 100644
--- a/app/models/user.rb
+++ b/app/models/user.rb
@@ -169,7 +169,7 @@ class User < ApplicationRecord
   end
 
   def functional?
-    confirmed? && approved? && !disabled? && !account.suspended? && account.moved_to_account_id.nil?
+    confirmed? && approved? && !disabled? && !account.suspended?
   end
 
   def unconfirmed_or_pending?
-- 
cgit 


From d8f96028c54bb47e6edddbd936bc8f2301dc9fa3 Mon Sep 17 00:00:00 2001
From: ThibG <thib@sitedethib.com>
Date: Sat, 30 Nov 2019 19:53:58 +0100
Subject: Add ability to filter reports by target account domain (#12154)

* Add ability to filter reports by target account domain

* Reword by_target_domain label
---
 app/controllers/admin/reports_controller.rb |  3 ++-
 app/helpers/admin/filter_helper.rb          |  2 +-
 app/models/report_filter.rb                 |  2 ++
 app/views/admin/reports/index.html.haml     | 14 ++++++++++++++
 config/locales/en.yml                       |  1 +
 5 files changed, 20 insertions(+), 2 deletions(-)

(limited to 'app/models')

diff --git a/app/controllers/admin/reports_controller.rb b/app/controllers/admin/reports_controller.rb
index f138376b2..09ce1761c 100644
--- a/app/controllers/admin/reports_controller.rb
+++ b/app/controllers/admin/reports_controller.rb
@@ -55,7 +55,8 @@ module Admin
       params.permit(
         :account_id,
         :resolved,
-        :target_account_id
+        :target_account_id,
+        :by_target_domain
       )
     end
 
diff --git a/app/helpers/admin/filter_helper.rb b/app/helpers/admin/filter_helper.rb
index 8af1683e7..fc4f15985 100644
--- a/app/helpers/admin/filter_helper.rb
+++ b/app/helpers/admin/filter_helper.rb
@@ -2,7 +2,7 @@
 
 module Admin::FilterHelper
   ACCOUNT_FILTERS      = %i(local remote by_domain active pending silenced suspended username display_name email ip staff).freeze
-  REPORT_FILTERS       = %i(resolved account_id target_account_id).freeze
+  REPORT_FILTERS       = %i(resolved account_id target_account_id by_target_domain).freeze
   INVITE_FILTER        = %i(available expired).freeze
   CUSTOM_EMOJI_FILTERS = %i(local remote by_domain shortcode).freeze
   TAGS_FILTERS         = %i(directory reviewed unreviewed pending_review popular active name).freeze
diff --git a/app/models/report_filter.rb b/app/models/report_filter.rb
index a392d60c3..abf53cbab 100644
--- a/app/models/report_filter.rb
+++ b/app/models/report_filter.rb
@@ -19,6 +19,8 @@ class ReportFilter
 
   def scope_for(key, value)
     case key.to_sym
+    when :by_target_domain
+      Report.where(target_account: Account.where(domain: value))
     when :resolved
       Report.resolved
     when :account_id
diff --git a/app/views/admin/reports/index.html.haml b/app/views/admin/reports/index.html.haml
index bfbd32108..b09472270 100644
--- a/app/views/admin/reports/index.html.haml
+++ b/app/views/admin/reports/index.html.haml
@@ -8,6 +8,20 @@
       %li= filter_link_to t('admin.reports.unresolved'), resolved: nil
       %li= filter_link_to t('admin.reports.resolved'), resolved: '1'
 
+= form_tag admin_reports_url, method: 'GET', class: 'simple_form' do
+  .fields-group
+    - Admin::FilterHelper::REPORT_FILTERS.each do |key|
+      - if params[key].present?
+        = hidden_field_tag key, params[key]
+
+    - %i(by_target_domain).each do |key|
+      .input.string.optional
+        = text_field_tag key, params[key], class: 'string optional', placeholder: I18n.t("admin.reports.#{key}")
+
+    .actions
+      %button= t('admin.accounts.search')
+      = link_to t('admin.accounts.reset'), admin_reports_path, class: 'button negative'
+
 - @reports.group_by(&:target_account_id).each do |target_account_id, reports|
   - target_account = reports.first.target_account
   .report-card
diff --git a/config/locales/en.yml b/config/locales/en.yml
index 783b7a4f6..e69b3596f 100644
--- a/config/locales/en.yml
+++ b/config/locales/en.yml
@@ -405,6 +405,7 @@ en:
       are_you_sure: Are you sure?
       assign_to_self: Assign to me
       assigned: Assigned moderator
+      by_target_domain: Domain of reported account
       comment:
         none: None
       created_at: Reported
-- 
cgit 


From d70268f0991ba69568112d4da5768e821d5983dd Mon Sep 17 00:00:00 2001
From: Mathieu Brunot <mathieu.brunot@monogramm.io>
Date: Sun, 1 Dec 2019 07:21:28 +0100
Subject: :sparkles: Convert LDAP username (#12461)

* :sparkles: Convert LDAP username #12021

Signed-off-by: mathieu.brunot <mathieu.brunot@monogramm.io>

* :bug: Fix conversion var use

Signed-off-by: mathieu.brunot <mathieu.brunot@monogramm.io>

* :bug: Fix LDAP uid conversion test

Signed-off-by: mathieu.brunot <mathieu.brunot@monogramm.io>

* :ok_hand: Remove comments with ref to PR

Signed-off-by: mathieu.brunot <mathieu.brunot@monogramm.io>

* :ok_hand: Remove unnecessary paranthesis

Signed-off-by: mathieu.brunot <mathieu.brunot@monogramm.io>

* :wrench: Move space in conversion string

Signed-off-by: mathieu.brunot <mathieu.brunot@monogramm.io>
---
 .env.nanobox                              |  3 +++
 .env.production.sample                    |  3 +++
 app/models/concerns/ldap_authenticable.rb | 12 ++++++++++--
 config/initializers/devise.rb             |  9 +++++++++
 4 files changed, 25 insertions(+), 2 deletions(-)

(limited to 'app/models')

diff --git a/.env.nanobox b/.env.nanobox
index cfbe487fb..fc6c3c42f 100644
--- a/.env.nanobox
+++ b/.env.nanobox
@@ -183,6 +183,9 @@ SMTP_FROM_ADDRESS=notifications@${APP_NAME}.nanoapp.io
 # LDAP_BIND_DN=
 # LDAP_PASSWORD=
 # LDAP_UID=cn
+# LDAP_UID_CONVERSION_ENABLED=true
+# LDAP_UID_CONVERSION_SEARCH=., -
+# LDAP_UID_CONVERSION_REPLACE=_
 
 # PAM authentication (optional)
 # PAM authentication uses for the email generation the "email" pam variable
diff --git a/.env.production.sample b/.env.production.sample
index f9a8bb7c1..6b078c7b2 100644
--- a/.env.production.sample
+++ b/.env.production.sample
@@ -179,6 +179,9 @@ STREAMING_CLUSTER_NUM=1
 # LDAP_PASSWORD=
 # LDAP_UID=cn
 # LDAP_SEARCH_FILTER=%{uid}=%{email}
+# LDAP_UID_CONVERSION_ENABLED=true
+# LDAP_UID_CONVERSION_SEARCH=., -
+# LDAP_UID_CONVERSION_REPLACE=_
 
 # PAM authentication (optional)
 # PAM authentication uses for the email generation the "email" pam variable
diff --git a/app/models/concerns/ldap_authenticable.rb b/app/models/concerns/ldap_authenticable.rb
index 117993947..2d2e1edbb 100644
--- a/app/models/concerns/ldap_authenticable.rb
+++ b/app/models/concerns/ldap_authenticable.rb
@@ -14,10 +14,18 @@ module LdapAuthenticable
     end
 
     def ldap_get_user(attributes = {})
-      resource = joins(:account).find_by(accounts: { username: attributes[Devise.ldap_uid.to_sym].first })
+      safe_username = attributes[Devise.ldap_uid.to_sym].first
+      if Devise.ldap_uid_conversion_enabled
+        keys = Regexp.union(Devise.ldap_uid_conversion_search.chars)
+        replacement = Devise.ldap_uid_conversion_replace
+
+        safe_username = safe_username.gsub(keys, replacement)
+      end
+
+      resource = joins(:account).find_by(accounts: { username: safe_username })
 
       if resource.blank?
-        resource = new(email: attributes[:mail].first, agreement: true, account_attributes: { username: attributes[Devise.ldap_uid.to_sym].first }, admin: false, external: true, confirmed_at: Time.now.utc)
+        resource = new(email: attributes[:mail].first, agreement: true, account_attributes: { username: safe_username }, admin: false, external: true, confirmed_at: Time.now.utc)
         resource.save!
       end
 
diff --git a/config/initializers/devise.rb b/config/initializers/devise.rb
index fd9a5a8b9..fa9fd8cc4 100644
--- a/config/initializers/devise.rb
+++ b/config/initializers/devise.rb
@@ -61,6 +61,12 @@ module Devise
   @@ldap_tls_no_verify = false
   mattr_accessor :ldap_search_filter
   @@ldap_search_filter = nil
+  mattr_accessor :ldap_uid_conversion_enabled
+  @@ldap_uid_conversion_enabled = false
+  mattr_accessor :ldap_uid_conversion_search
+  @@ldap_uid_conversion_search = nil
+  mattr_accessor :ldap_uid_conversion_replace
+  @@ldap_uid_conversion_replace = nil
 
   class Strategies::PamAuthenticatable
     def valid?
@@ -365,5 +371,8 @@ Devise.setup do |config|
     config.ldap_uid            = ENV.fetch('LDAP_UID', 'cn')
     config.ldap_tls_no_verify  = ENV['LDAP_TLS_NO_VERIFY'] == 'true'
     config.ldap_search_filter  = ENV.fetch('LDAP_SEARCH_FILTER', '%{uid}=%{email}')
+    config.ldap_uid_conversion_enabled  = ENV['LDAP_UID_CONVERSION_ENABLED'] == 'true'
+    config.ldap_uid_conversion_search   = ENV.fetch('LDAP_UID_CONVERSION_SEARCH', '.,- ')
+    config.ldap_uid_conversion_replace  = ENV.fetch('LDAP_UID_CONVERSION_REPLACE', '_')
   end
 end
-- 
cgit 


From f60cd97638c9de9cbc4714cfbc87a321324fe9ff Mon Sep 17 00:00:00 2001
From: ThibG <thib@sitedethib.com>
Date: Sun, 1 Dec 2019 17:24:33 +0100
Subject: Only normalize local polls (#12515)

Before this patch, if remote poll options have leading or trailing spaces,
the information stored locally won't match them, causing federated voting to
fail.
---
 app/models/poll.rb | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

(limited to 'app/models')

diff --git a/app/models/poll.rb b/app/models/poll.rb
index 5427368fd..b5deafcc2 100644
--- a/app/models/poll.rb
+++ b/app/models/poll.rb
@@ -36,7 +36,7 @@ class Poll < ApplicationRecord
   scope :attached, -> { where.not(status_id: nil) }
   scope :unattached, -> { where(status_id: nil) }
 
-  before_validation :prepare_options
+  before_validation :prepare_options, if: :local?
   before_validation :prepare_votes_count
 
   after_initialize :prepare_cached_tallies
-- 
cgit 


From 911cc144815babf83ddf99f2daa3682021d401b8 Mon Sep 17 00:00:00 2001
From: ThibG <thib@sitedethib.com>
Date: Sun, 1 Dec 2019 17:25:29 +0100
Subject: Add follow_request notification type (#12198)

* Add follow_request notification type

The notification type already existed in the backend but was never pushed
to the front-end. This also means translation strings were also available
for the backend, from the notification mailer.

Unlike other notification types, these are off by default, to match what
I remember of Gargron's view on the topic: that follow requests should not
clutter notifications and should instead be reviewed at the user's own
leisure in the dedicated column.

Since follow requests have their own column, I've deemed it unnecessary to
add a specific tab for them in the notification quick filter.

* Show follow request link in single-column if there are pending requests, even if account isn't locked

* Push follow requests from notifications to the follow_requests list

* Offer to accept or reject follow request from the notification

* Redesign follow request notification
---
 .../api/v1/push/subscriptions_controller.rb        |  2 +-
 .../api/web/push_subscriptions_controller.rb       |  3 +-
 app/javascript/mastodon/actions/notifications.js   |  2 +-
 .../notifications/components/column_settings.js    | 11 ++++
 .../notifications/components/follow_request.js     | 59 ++++++++++++++++++++++
 .../notifications/components/notification.js       | 27 +++++++++-
 .../containers/follow_request_container.js         | 26 ++++++++++
 .../ui/components/follow_requests_nav_link.js      | 13 ++---
 app/javascript/mastodon/reducers/notifications.js  | 11 +++-
 .../mastodon/reducers/push_notifications.js        |  1 +
 app/javascript/mastodon/reducers/settings.js       |  3 ++
 app/javascript/mastodon/reducers/user_lists.js     | 11 ++++
 .../mastodon/service_worker/web_push_locales.js    |  1 +
 app/models/notification.rb                         |  8 +--
 app/services/notify_service.rb                     |  2 +-
 spec/models/notification_spec.rb                   | 26 ----------
 16 files changed, 158 insertions(+), 48 deletions(-)
 create mode 100644 app/javascript/mastodon/features/notifications/components/follow_request.js
 create mode 100644 app/javascript/mastodon/features/notifications/containers/follow_request_container.js

(limited to 'app/models')

diff --git a/app/controllers/api/v1/push/subscriptions_controller.rb b/app/controllers/api/v1/push/subscriptions_controller.rb
index 1b658f870..1cbc92b93 100644
--- a/app/controllers/api/v1/push/subscriptions_controller.rb
+++ b/app/controllers/api/v1/push/subscriptions_controller.rb
@@ -51,6 +51,6 @@ class Api::V1::Push::SubscriptionsController < Api::BaseController
 
   def data_params
     return {} if params[:data].blank?
-    params.require(:data).permit(alerts: [:follow, :favourite, :reblog, :mention, :poll])
+    params.require(:data).permit(alerts: [:follow, :follow_request, :favourite, :reblog, :mention, :poll])
   end
 end
diff --git a/app/controllers/api/web/push_subscriptions_controller.rb b/app/controllers/api/web/push_subscriptions_controller.rb
index d8153e082..f388b17e5 100644
--- a/app/controllers/api/web/push_subscriptions_controller.rb
+++ b/app/controllers/api/web/push_subscriptions_controller.rb
@@ -19,6 +19,7 @@ class Api::Web::PushSubscriptionsController < Api::Web::BaseController
     data = {
       alerts: {
         follow: alerts_enabled,
+        follow_request: false,
         favourite: alerts_enabled,
         reblog: alerts_enabled,
         mention: alerts_enabled,
@@ -58,6 +59,6 @@ class Api::Web::PushSubscriptionsController < Api::Web::BaseController
   end
 
   def data_params
-    @data_params ||= params.require(:data).permit(alerts: [:follow, :favourite, :reblog, :mention, :poll])
+    @data_params ||= params.require(:data).permit(alerts: [:follow, :follow_request, :favourite, :reblog, :mention, :poll])
   end
 end
diff --git a/app/javascript/mastodon/actions/notifications.js b/app/javascript/mastodon/actions/notifications.js
index 3a92e0224..798f9b37e 100644
--- a/app/javascript/mastodon/actions/notifications.js
+++ b/app/javascript/mastodon/actions/notifications.js
@@ -110,7 +110,7 @@ export function updateNotifications(notification, intlMessages, intlLocale) {
 const excludeTypesFromSettings = state => state.getIn(['settings', 'notifications', 'shows']).filter(enabled => !enabled).keySeq().toJS();
 
 const excludeTypesFromFilter = filter => {
-  const allTypes = ImmutableList(['follow', 'favourite', 'reblog', 'mention', 'poll']);
+  const allTypes = ImmutableList(['follow', 'follow_request', 'favourite', 'reblog', 'mention', 'poll']);
   return allTypes.filterNot(item => item === filter).toJS();
 };
 
diff --git a/app/javascript/mastodon/features/notifications/components/column_settings.js b/app/javascript/mastodon/features/notifications/components/column_settings.js
index 60a86312a..8bd03fbda 100644
--- a/app/javascript/mastodon/features/notifications/components/column_settings.js
+++ b/app/javascript/mastodon/features/notifications/components/column_settings.js
@@ -57,6 +57,17 @@ export default class ColumnSettings extends React.PureComponent {
           </div>
         </div>
 
+        <div role='group' aria-labelledby='notifications-follow-request'>
+          <span id='notifications-follow-request' className='column-settings__section'><FormattedMessage id='notifications.column_settings.follow_request' defaultMessage='New follow requests:' /></span>
+
+          <div className='column-settings__row'>
+            <SettingToggle prefix='notifications_desktop' settings={settings} settingPath={['alerts', 'follow_request']} onChange={onChange} label={alertStr} />
+            {showPushSettings && <SettingToggle prefix='notifications_push' settings={pushSettings} settingPath={['alerts', 'follow_request']} onChange={this.onPushChange} label={pushStr} />}
+            <SettingToggle prefix='notifications' settings={settings} settingPath={['shows', 'follow_request']} onChange={onChange} label={showStr} />
+            <SettingToggle prefix='notifications' settings={settings} settingPath={['sounds', 'follow_request']} onChange={onChange} label={soundStr} />
+          </div>
+        </div>
+
         <div role='group' aria-labelledby='notifications-favourite'>
           <span id='notifications-favourite' className='column-settings__section'><FormattedMessage id='notifications.column_settings.favourite' defaultMessage='Favourites:' /></span>
 
diff --git a/app/javascript/mastodon/features/notifications/components/follow_request.js b/app/javascript/mastodon/features/notifications/components/follow_request.js
new file mode 100644
index 000000000..a80cfb2fa
--- /dev/null
+++ b/app/javascript/mastodon/features/notifications/components/follow_request.js
@@ -0,0 +1,59 @@
+import React, { Fragment } from 'react';
+import ImmutablePropTypes from 'react-immutable-proptypes';
+import PropTypes from 'prop-types';
+import Avatar from 'mastodon/components/avatar';
+import DisplayName from 'mastodon/components/display_name';
+import Permalink from 'mastodon/components/permalink';
+import IconButton from 'mastodon/components/icon_button';
+import { defineMessages, injectIntl } from 'react-intl';
+import ImmutablePureComponent from 'react-immutable-pure-component';
+
+const messages = defineMessages({
+  authorize: { id: 'follow_request.authorize', defaultMessage: 'Authorize' },
+  reject: { id: 'follow_request.reject', defaultMessage: 'Reject' },
+});
+
+export default @injectIntl
+class FollowRequest extends ImmutablePureComponent {
+
+  static propTypes = {
+    account: ImmutablePropTypes.map.isRequired,
+    onAuthorize: PropTypes.func.isRequired,
+    onReject: PropTypes.func.isRequired,
+    intl: PropTypes.object.isRequired,
+  };
+
+  render () {
+    const { intl, hidden, account, onAuthorize, onReject } = this.props;
+
+    if (!account) {
+      return <div />;
+    }
+
+    if (hidden) {
+      return (
+        <Fragment>
+          {account.get('display_name')}
+          {account.get('username')}
+        </Fragment>
+      );
+    }
+
+    return (
+      <div className='account'>
+        <div className='account__wrapper'>
+          <Permalink key={account.get('id')} className='account__display-name' title={account.get('acct')} href={account.get('url')} to={`/accounts/${account.get('id')}`}>
+            <div className='account__avatar-wrapper'><Avatar account={account} size={36} /></div>
+            <DisplayName account={account} />
+          </Permalink>
+
+          <div className='account__relationship'>
+            <IconButton title={intl.formatMessage(messages.authorize)} icon='check' onClick={onAuthorize} />
+            <IconButton title={intl.formatMessage(messages.reject)} icon='times' onClick={onReject} />
+          </div>
+        </div>
+      </div>
+    );
+  }
+
+}
diff --git a/app/javascript/mastodon/features/notifications/components/notification.js b/app/javascript/mastodon/features/notifications/components/notification.js
index 2dea8afa7..74065e5e2 100644
--- a/app/javascript/mastodon/features/notifications/components/notification.js
+++ b/app/javascript/mastodon/features/notifications/components/notification.js
@@ -7,6 +7,7 @@ import ImmutablePureComponent from 'react-immutable-pure-component';
 import { me } from 'mastodon/initial_state';
 import StatusContainer from 'mastodon/containers/status_container';
 import AccountContainer from 'mastodon/containers/account_container';
+import FollowRequestContainer from '../containers/follow_request_container';
 import Icon from 'mastodon/components/icon';
 import Permalink from 'mastodon/components/permalink';
 
@@ -127,7 +128,29 @@ class Notification extends ImmutablePureComponent {
             </span>
           </div>
 
-          <AccountContainer id={account.get('id')} withNote={false} hidden={this.props.hidden} />
+          <AccountContainer id={account.get('id')} hidden={this.props.hidden} />
+        </div>
+      </HotKeys>
+    );
+  }
+
+  renderFollowRequest (notification, account, link) {
+    const { intl } = this.props;
+
+    return (
+      <HotKeys handlers={this.getHandlers()}>
+        <div className='notification notification-follow-request focusable' tabIndex='0' aria-label={notificationForScreenReader(intl, intl.formatMessage({ id: 'notification.follow_request', defaultMessage: '{name} has requested to follow you' }, { name: account.get('acct') }), notification.get('created_at'))}>
+          <div className='notification__message'>
+            <div className='notification__favourite-icon-wrapper'>
+              <Icon id='user' fixedWidth />
+            </div>
+
+            <span title={notification.get('created_at')}>
+              <FormattedMessage id='notification.follow_request' defaultMessage='{name} has requested to follow you' values={{ name: link }} />
+            </span>
+          </div>
+
+          <FollowRequestContainer id={account.get('id')} withNote={false} hidden={this.props.hidden} />
         </div>
       </HotKeys>
     );
@@ -261,6 +284,8 @@ class Notification extends ImmutablePureComponent {
     switch(notification.get('type')) {
     case 'follow':
       return this.renderFollow(notification, account, link);
+    case 'follow_request':
+      return this.renderFollowRequest(notification, account, link);
     case 'mention':
       return this.renderMention(notification);
     case 'favourite':
diff --git a/app/javascript/mastodon/features/notifications/containers/follow_request_container.js b/app/javascript/mastodon/features/notifications/containers/follow_request_container.js
new file mode 100644
index 000000000..f9f6c577e
--- /dev/null
+++ b/app/javascript/mastodon/features/notifications/containers/follow_request_container.js
@@ -0,0 +1,26 @@
+import { connect } from 'react-redux';
+import { makeGetAccount } from 'mastodon/selectors';
+import FollowRequest from '../components/follow_request';
+import { authorizeFollowRequest, rejectFollowRequest } from 'mastodon/actions/accounts';
+
+const makeMapStateToProps = () => {
+  const getAccount = makeGetAccount();
+
+  const mapStateToProps = (state, props) => ({
+    account: getAccount(state, props.id),
+  });
+
+  return mapStateToProps;
+};
+
+const mapDispatchToProps = (dispatch, { id }) => ({
+  onAuthorize () {
+    dispatch(authorizeFollowRequest(id));
+  },
+
+  onReject () {
+    dispatch(rejectFollowRequest(id));
+  },
+});
+
+export default connect(makeMapStateToProps, mapDispatchToProps)(FollowRequest);
diff --git a/app/javascript/mastodon/features/ui/components/follow_requests_nav_link.js b/app/javascript/mastodon/features/ui/components/follow_requests_nav_link.js
index 90c953893..950ed7b27 100644
--- a/app/javascript/mastodon/features/ui/components/follow_requests_nav_link.js
+++ b/app/javascript/mastodon/features/ui/components/follow_requests_nav_link.js
@@ -4,12 +4,10 @@ import { fetchFollowRequests } from 'mastodon/actions/accounts';
 import { connect } from 'react-redux';
 import { NavLink, withRouter } from 'react-router-dom';
 import IconWithBadge from 'mastodon/components/icon_with_badge';
-import { me } from 'mastodon/initial_state';
 import { List as ImmutableList } from 'immutable';
 import { FormattedMessage } from 'react-intl';
 
 const mapStateToProps = state => ({
-  locked: state.getIn(['accounts', me, 'locked']),
   count: state.getIn(['user_lists', 'follow_requests', 'items'], ImmutableList()).size,
 });
 
@@ -19,22 +17,19 @@ class FollowRequestsNavLink extends React.Component {
 
   static propTypes = {
     dispatch: PropTypes.func.isRequired,
-    locked: PropTypes.bool,
     count: PropTypes.number.isRequired,
   };
 
   componentDidMount () {
-    const { dispatch, locked } = this.props;
+    const { dispatch } = this.props;
 
-    if (locked) {
-      dispatch(fetchFollowRequests());
-    }
+    dispatch(fetchFollowRequests());
   }
 
   render () {
-    const { locked, count } = this.props;
+    const { count } = this.props;
 
-    if (!locked || count === 0) {
+    if (count === 0) {
       return null;
     }
 
diff --git a/app/javascript/mastodon/reducers/notifications.js b/app/javascript/mastodon/reducers/notifications.js
index 6ba80bd6a..60e901e39 100644
--- a/app/javascript/mastodon/reducers/notifications.js
+++ b/app/javascript/mastodon/reducers/notifications.js
@@ -13,6 +13,8 @@ import {
 import {
   ACCOUNT_BLOCK_SUCCESS,
   ACCOUNT_MUTE_SUCCESS,
+  FOLLOW_REQUEST_AUTHORIZE_SUCCESS,
+  FOLLOW_REQUEST_REJECT_SUCCESS,
 } from '../actions/accounts';
 import { DOMAIN_BLOCK_SUCCESS } from 'mastodon/actions/domain_blocks';
 import { TIMELINE_DELETE, TIMELINE_DISCONNECT } from '../actions/timelines';
@@ -89,8 +91,8 @@ const expandNormalizedNotifications = (state, notifications, next, isLoadingRece
   });
 };
 
-const filterNotifications = (state, accountIds) => {
-  const helper = list => list.filterNot(item => item !== null && accountIds.includes(item.get('account')));
+const filterNotifications = (state, accountIds, type) => {
+  const helper = list => list.filterNot(item => item !== null && accountIds.includes(item.get('account')) && (type === undefined || type === item.get('type')));
   return state.update('items', helper).update('pendingItems', helper);
 };
 
@@ -129,6 +131,11 @@ export default function notifications(state = initialState, action) {
     return action.relationship.muting_notifications ? filterNotifications(state, [action.relationship.id]) : state;
   case DOMAIN_BLOCK_SUCCESS:
     return filterNotifications(state, action.accounts);
+  case FOLLOW_REQUEST_AUTHORIZE_SUCCESS:
+  case FOLLOW_REQUEST_REJECT_SUCCESS:
+    return filterNotifications(state, [action.id], 'follow_request');
+  case ACCOUNT_MUTE_SUCCESS:
+    return action.relationship.muting_notifications ? filterNotifications(state, [action.relationship.id]) : state;
   case NOTIFICATIONS_CLEAR:
     return state.set('items', ImmutableList()).set('pendingItems', ImmutableList()).set('hasMore', false);
   case TIMELINE_DELETE:
diff --git a/app/javascript/mastodon/reducers/push_notifications.js b/app/javascript/mastodon/reducers/push_notifications.js
index 317352b79..c48cfb705 100644
--- a/app/javascript/mastodon/reducers/push_notifications.js
+++ b/app/javascript/mastodon/reducers/push_notifications.js
@@ -6,6 +6,7 @@ const initialState = Immutable.Map({
   subscription: null,
   alerts: new Immutable.Map({
     follow: false,
+    follow_request: false,
     favourite: false,
     reblog: false,
     mention: false,
diff --git a/app/javascript/mastodon/reducers/settings.js b/app/javascript/mastodon/reducers/settings.js
index 793a99f8f..efef2ad9a 100644
--- a/app/javascript/mastodon/reducers/settings.js
+++ b/app/javascript/mastodon/reducers/settings.js
@@ -30,6 +30,7 @@ const initialState = ImmutableMap({
   notifications: ImmutableMap({
     alerts: ImmutableMap({
       follow: true,
+      follow_request: false,
       favourite: true,
       reblog: true,
       mention: true,
@@ -44,6 +45,7 @@ const initialState = ImmutableMap({
 
     shows: ImmutableMap({
       follow: true,
+      follow_request: false,
       favourite: true,
       reblog: true,
       mention: true,
@@ -52,6 +54,7 @@ const initialState = ImmutableMap({
 
     sounds: ImmutableMap({
       follow: true,
+      follow_request: false,
       favourite: true,
       reblog: true,
       mention: true,
diff --git a/app/javascript/mastodon/reducers/user_lists.js b/app/javascript/mastodon/reducers/user_lists.js
index 08e94022f..a7853452f 100644
--- a/app/javascript/mastodon/reducers/user_lists.js
+++ b/app/javascript/mastodon/reducers/user_lists.js
@@ -1,3 +1,6 @@
+import {
+  NOTIFICATIONS_UPDATE,
+} from '../actions/notifications';
 import {
   FOLLOWERS_FETCH_SUCCESS,
   FOLLOWERS_EXPAND_SUCCESS,
@@ -53,6 +56,12 @@ const appendToList = (state, type, id, accounts, next) => {
   });
 };
 
+const normalizeFollowRequest = (state, notification) => {
+  return state.updateIn(['follow_requests', 'items'], list => {
+    return list.filterNot(item => item === notification.account.id).unshift(notification.account.id);
+  });
+};
+
 export default function userLists(state = initialState, action) {
   switch(action.type) {
   case FOLLOWERS_FETCH_SUCCESS:
@@ -67,6 +76,8 @@ export default function userLists(state = initialState, action) {
     return state.setIn(['reblogged_by', action.id], ImmutableList(action.accounts.map(item => item.id)));
   case FAVOURITES_FETCH_SUCCESS:
     return state.setIn(['favourited_by', action.id], ImmutableList(action.accounts.map(item => item.id)));
+  case NOTIFICATIONS_UPDATE:
+    return action.notification.type === 'follow_request' ? normalizeFollowRequest(state, action.notification) : state;
   case FOLLOW_REQUESTS_FETCH_SUCCESS:
     return state.setIn(['follow_requests', 'items'], ImmutableList(action.accounts.map(item => item.id))).setIn(['follow_requests', 'next'], action.next);
   case FOLLOW_REQUESTS_EXPAND_SUCCESS:
diff --git a/app/javascript/mastodon/service_worker/web_push_locales.js b/app/javascript/mastodon/service_worker/web_push_locales.js
index 5ce8c7b50..1265f3cfa 100644
--- a/app/javascript/mastodon/service_worker/web_push_locales.js
+++ b/app/javascript/mastodon/service_worker/web_push_locales.js
@@ -16,6 +16,7 @@ filenames.forEach(filename => {
   filtered[locale] = {
     'notification.favourite': full['notification.favourite'] || '',
     'notification.follow': full['notification.follow'] || '',
+    'notification.follow_request': full['notification.follow_request'] || '',
     'notification.mention': full['notification.mention'] || '',
     'notification.reblog': full['notification.reblog'] || '',
     'notification.poll': full['notification.poll'] || '',
diff --git a/app/models/notification.rb b/app/models/notification.rb
index 498673ff1..ad7528f50 100644
--- a/app/models/notification.rb
+++ b/app/models/notification.rb
@@ -42,7 +42,7 @@ class Notification < ApplicationRecord
   validates :activity_type, inclusion: { in: TYPE_CLASS_MAP.values }
 
   scope :browserable, ->(exclude_types = [], account_id = nil) {
-    types = TYPE_CLASS_MAP.values - activity_types_from_types(exclude_types + [:follow_request])
+    types = TYPE_CLASS_MAP.values - activity_types_from_types(exclude_types)
     if account_id.nil?
       where(activity_type: types)
     else
@@ -50,7 +50,7 @@ class Notification < ApplicationRecord
     end
   }
 
-  cache_associated :from_account, status: STATUS_INCLUDES, mention: [status: STATUS_INCLUDES], favourite: [:account, status: STATUS_INCLUDES], follow: :account, poll: [status: STATUS_INCLUDES]
+  cache_associated :from_account, status: STATUS_INCLUDES, mention: [status: STATUS_INCLUDES], favourite: [:account, status: STATUS_INCLUDES], follow: :account, follow_request: :account, poll: [status: STATUS_INCLUDES]
 
   def type
     @type ||= TYPE_CLASS_MAP.invert[activity_type].to_sym
@@ -69,10 +69,6 @@ class Notification < ApplicationRecord
     end
   end
 
-  def browserable?
-    type != :follow_request
-  end
-
   class << self
     def cache_ids
       select(:id, :updated_at, :activity_type, :activity_id)
diff --git a/app/services/notify_service.rb b/app/services/notify_service.rb
index b5c721589..9364a6ae8 100644
--- a/app/services/notify_service.rb
+++ b/app/services/notify_service.rb
@@ -9,7 +9,7 @@ class NotifyService < BaseService
     return if recipient.user.nil? || blocked?
 
     create_notification!
-    push_notification! if @notification.browserable?
+    push_notification!
     push_to_conversation! if direct_message?
     send_email! if email_enabled?
   rescue ActiveRecord::RecordInvalid
diff --git a/spec/models/notification_spec.rb b/spec/models/notification_spec.rb
index 59c582cde..d2e676ec2 100644
--- a/spec/models/notification_spec.rb
+++ b/spec/models/notification_spec.rb
@@ -34,32 +34,6 @@ RSpec.describe Notification, type: :model do
     end
   end
 
-  describe '#browserable?' do
-    let(:notification) { Fabricate(:notification) }
-
-    subject { notification.browserable? }
-
-    context 'type is :follow_request' do
-      before do
-        allow(notification).to receive(:type).and_return(:follow_request)
-      end
-
-      it 'returns false' do
-        is_expected.to be false
-      end
-    end
-
-    context 'type is not :follow_request' do
-      before do
-        allow(notification).to receive(:type).and_return(:else)
-      end
-
-      it 'returns true' do
-        is_expected.to be true
-      end
-    end
-  end
-
   describe '#type' do
     it 'returns :reblog for a Status' do
       notification = Notification.new(activity: Status.new)
-- 
cgit 


From bd8dc9bd0c9857470f14189c15572cab18bf8ab1 Mon Sep 17 00:00:00 2001
From: Mathieu Brunot <mathieu.brunot@monogramm.io>
Date: Sun, 1 Dec 2019 18:52:21 +0100
Subject: :sparkles: Add an LDAP Mail attribute config (#12053)

Signed-off-by: mathieu.brunot <mathieu.brunot@monogramm.io>
---
 .env.nanobox                              | 2 ++
 .env.production.sample                    | 3 ++-
 app/models/concerns/ldap_authenticable.rb | 4 ++--
 config/initializers/devise.rb             | 5 ++++-
 4 files changed, 10 insertions(+), 4 deletions(-)

(limited to 'app/models')

diff --git a/.env.nanobox b/.env.nanobox
index fc6c3c42f..03aa01a34 100644
--- a/.env.nanobox
+++ b/.env.nanobox
@@ -183,6 +183,8 @@ SMTP_FROM_ADDRESS=notifications@${APP_NAME}.nanoapp.io
 # LDAP_BIND_DN=
 # LDAP_PASSWORD=
 # LDAP_UID=cn
+# LDAP_MAIL=mail
+# LDAP_SEARCH_FILTER=(|(%{uid}=%{email})(%{mail}=%{email}))
 # LDAP_UID_CONVERSION_ENABLED=true
 # LDAP_UID_CONVERSION_SEARCH=., -
 # LDAP_UID_CONVERSION_REPLACE=_
diff --git a/.env.production.sample b/.env.production.sample
index 6b078c7b2..9cab992e3 100644
--- a/.env.production.sample
+++ b/.env.production.sample
@@ -178,7 +178,8 @@ STREAMING_CLUSTER_NUM=1
 # LDAP_BIND_DN=
 # LDAP_PASSWORD=
 # LDAP_UID=cn
-# LDAP_SEARCH_FILTER=%{uid}=%{email}
+# LDAP_MAIL=mail
+# LDAP_SEARCH_FILTER=(|(%{uid}=%{email})(%{mail}=%{email}))
 # LDAP_UID_CONVERSION_ENABLED=true
 # LDAP_UID_CONVERSION_SEARCH=., -
 # LDAP_UID_CONVERSION_REPLACE=_
diff --git a/app/models/concerns/ldap_authenticable.rb b/app/models/concerns/ldap_authenticable.rb
index 2d2e1edbb..e3f94bb6c 100644
--- a/app/models/concerns/ldap_authenticable.rb
+++ b/app/models/concerns/ldap_authenticable.rb
@@ -6,7 +6,7 @@ module LdapAuthenticable
   class_methods do
     def authenticate_with_ldap(params = {})
       ldap   = Net::LDAP.new(ldap_options)
-      filter = format(Devise.ldap_search_filter, uid: Devise.ldap_uid, email: params[:email])
+      filter = format(Devise.ldap_search_filter, uid: Devise.ldap_uid, mail: Devise.ldap_mail, email: params[:email])
 
       if (user_info = ldap.bind_as(base: Devise.ldap_base, filter: filter, password: params[:password]))
         ldap_get_user(user_info.first)
@@ -25,7 +25,7 @@ module LdapAuthenticable
       resource = joins(:account).find_by(accounts: { username: safe_username })
 
       if resource.blank?
-        resource = new(email: attributes[:mail].first, agreement: true, account_attributes: { username: safe_username }, admin: false, external: true, confirmed_at: Time.now.utc)
+        resource = new(email: attributes[Devise.ldap_mail.to_sym].first, agreement: true, account_attributes: { username: safe_username }, admin: false, external: true, confirmed_at: Time.now.utc)
         resource.save!
       end
 
diff --git a/config/initializers/devise.rb b/config/initializers/devise.rb
index fa9fd8cc4..59e69ad37 100644
--- a/config/initializers/devise.rb
+++ b/config/initializers/devise.rb
@@ -53,6 +53,8 @@ module Devise
   @@ldap_base = nil
   mattr_accessor :ldap_uid
   @@ldap_uid = nil
+  mattr_accessor :ldap_mail
+  @@ldap_mail = nil
   mattr_accessor :ldap_bind_dn
   @@ldap_bind_dn = nil
   mattr_accessor :ldap_password
@@ -369,8 +371,9 @@ Devise.setup do |config|
     config.ldap_bind_dn        = ENV.fetch('LDAP_BIND_DN')
     config.ldap_password       = ENV.fetch('LDAP_PASSWORD')
     config.ldap_uid            = ENV.fetch('LDAP_UID', 'cn')
+    config.ldap_mail           = ENV.fetch('LDAP_MAIL', 'mail')
     config.ldap_tls_no_verify  = ENV['LDAP_TLS_NO_VERIFY'] == 'true'
-    config.ldap_search_filter  = ENV.fetch('LDAP_SEARCH_FILTER', '%{uid}=%{email}')
+    config.ldap_search_filter  = ENV.fetch('LDAP_SEARCH_FILTER', '(|(%{uid}=%{email})(%{mail}=%{email}))')
     config.ldap_uid_conversion_enabled  = ENV['LDAP_UID_CONVERSION_ENABLED'] == 'true'
     config.ldap_uid_conversion_search   = ENV.fetch('LDAP_UID_CONVERSION_SEARCH', '.,- ')
     config.ldap_uid_conversion_replace  = ENV.fetch('LDAP_UID_CONVERSION_REPLACE', '_')
-- 
cgit 


From 6be16d02cb3f03d21226f9b6664d33e3e8650043 Mon Sep 17 00:00:00 2001
From: Yamagishi Kazutoshi <ykzts@desire.sh>
Date: Tue, 3 Dec 2019 02:25:43 +0900
Subject: Update ESLint and RuboCop in Code Climate (#12534)

---
 .codeclimate.yml                | 4 ++--
 .rubocop.yml                    | 6 ++++++
 app/lib/activitypub/adapter.rb  | 1 +
 app/lib/language_detector.rb    | 2 +-
 app/models/media_attachment.rb  | 2 +-
 lib/paperclip/lazy_thumbnail.rb | 4 ++--
 6 files changed, 13 insertions(+), 6 deletions(-)

(limited to 'app/models')

diff --git a/.codeclimate.yml b/.codeclimate.yml
index 571507a54..9817d7f1c 100644
--- a/.codeclimate.yml
+++ b/.codeclimate.yml
@@ -27,10 +27,10 @@ plugins:
     enabled: true
   eslint:
     enabled: true
-    channel: eslint-5
+    channel: eslint-6
   rubocop:
     enabled: true
-    channel: rubocop-0-71
+    channel: rubocop-0-76
   sass-lint:
     enabled: true
 exclude_patterns:
diff --git a/.rubocop.yml b/.rubocop.yml
index 8bd4c867f..9a68becbb 100644
--- a/.rubocop.yml
+++ b/.rubocop.yml
@@ -71,6 +71,9 @@ Naming/MemoizedInstanceVariableName:
 Rails:
   Enabled: true
 
+Rails/EnumHash:
+  Enabled: false
+
 Rails/HasAndBelongsToMany:
   Enabled: false
 
@@ -102,6 +105,9 @@ Style/Documentation:
 Style/DoubleNegation:
   Enabled: true
 
+Style/FormatStringToken:
+  Enabled: false
+
 Style/FrozenStringLiteralComment:
   Enabled: true
 
diff --git a/app/lib/activitypub/adapter.rb b/app/lib/activitypub/adapter.rb
index 2a8f72333..78138fb73 100644
--- a/app/lib/activitypub/adapter.rb
+++ b/app/lib/activitypub/adapter.rb
@@ -35,6 +35,7 @@ class ActivityPub::Adapter < ActiveModelSerializers::Adapter::Base
   def serializable_hash(options = nil)
     named_contexts     = {}
     context_extensions = {}
+
     options         = serialization_options(options)
     serialized_hash = serializer.serializable_hash(options.merge(named_contexts: named_contexts, context_extensions: context_extensions))
     serialized_hash = serialized_hash.select { |k, _| options[:fields].include?(k) } if options[:fields]
diff --git a/app/lib/language_detector.rb b/app/lib/language_detector.rb
index 6f9511a54..302072bcc 100644
--- a/app/lib/language_detector.rb
+++ b/app/lib/language_detector.rb
@@ -44,7 +44,7 @@ class LanguageDetector
     words = text.scan(RELIABLE_CHARACTERS_RE)
 
     if words.present?
-      words.reduce(0) { |acc, elem| acc + elem.size }.to_f / text.size.to_f > 0.3
+      words.reduce(0) { |acc, elem| acc + elem.size }.to_f / text.size > 0.3
     else
       false
     end
diff --git a/app/models/media_attachment.rb b/app/models/media_attachment.rb
index 1f9d92b22..5d5034a52 100644
--- a/app/models/media_attachment.rb
+++ b/app/models/media_attachment.rb
@@ -287,7 +287,7 @@ class MediaAttachment < ApplicationRecord
       width:  width,
       height: height,
       size: "#{width}x#{height}",
-      aspect: width.to_f / height.to_f,
+      aspect: width.to_f / height,
     }
   end
 
diff --git a/lib/paperclip/lazy_thumbnail.rb b/lib/paperclip/lazy_thumbnail.rb
index 542c17fb2..10b14860c 100644
--- a/lib/paperclip/lazy_thumbnail.rb
+++ b/lib/paperclip/lazy_thumbnail.rb
@@ -9,8 +9,8 @@ module Paperclip
         min_side = [@current_geometry.width, @current_geometry.height].min.to_i
         options[:geometry] = "#{min_side}x#{min_side}#" if @target_geometry.square? && min_side < @target_geometry.width
       elsif options[:pixels]
-        width  = Math.sqrt(options[:pixels] * (@current_geometry.width.to_f / @current_geometry.height.to_f)).round.to_i
-        height = Math.sqrt(options[:pixels] * (@current_geometry.height.to_f / @current_geometry.width.to_f)).round.to_i
+        width  = Math.sqrt(options[:pixels] * (@current_geometry.width.to_f / @current_geometry.height)).round.to_i
+        height = Math.sqrt(options[:pixels] * (@current_geometry.height.to_f / @current_geometry.width)).round.to_i
         options[:geometry] = "#{width}x#{height}>"
       end
 
-- 
cgit 


From f43f1e01840cd0bad7a88c90d9ea44b183a2a15d Mon Sep 17 00:00:00 2001
From: Takeshi Umeda <noel.yoshiba@gmail.com>
Date: Thu, 5 Dec 2019 04:36:33 +0900
Subject: Add basic support for group actors (#12071)

* Show badge on group actor in WebUI

* Do not notify in case of  by following group actor

* If you mention group actor, also mention group actor followers

* Relax characters that can be used in username (same as Application)

* Revert "Relax characters that can be used in username (same as Application)"

This reverts commit 7e10a137b878d0db1b5252c52106faef5e09ca4b.

* Delete display_name method
---
 app/helpers/statuses_helper.rb                     | 68 ++++++++++++++++++++++
 .../mastodon/features/account/components/header.js | 11 +++-
 app/lib/activitypub/activity.rb                    |  6 +-
 app/lib/activitypub/tag_manager.rb                 | 30 ++++++++--
 app/models/account.rb                              |  7 +++
 app/serializers/activitypub/actor_serializer.rb    |  2 +
 app/serializers/rest/account_serializer.rb         |  2 +-
 config/locales/en.yml                              |  1 +
 8 files changed, 118 insertions(+), 9 deletions(-)

(limited to 'app/models')

diff --git a/app/helpers/statuses_helper.rb b/app/helpers/statuses_helper.rb
index 866a9902c..f0e3df944 100644
--- a/app/helpers/statuses_helper.rb
+++ b/app/helpers/statuses_helper.rb
@@ -4,6 +4,74 @@ module StatusesHelper
   EMBEDDED_CONTROLLER = 'statuses'
   EMBEDDED_ACTION = 'embed'
 
+  def account_action_button(account)
+    if user_signed_in?
+      if account.id == current_user.account_id
+        link_to settings_profile_url, class: 'button logo-button' do
+          safe_join([svg_logo, t('settings.edit_profile')])
+        end
+      elsif current_account.following?(account) || current_account.requested?(account)
+        link_to account_unfollow_path(account), class: 'button logo-button button--destructive', data: { method: :post } do
+          safe_join([svg_logo, t('accounts.unfollow')])
+        end
+      elsif !(account.memorial? || account.moved?)
+        link_to account_follow_path(account), class: "button logo-button#{account.blocking?(current_account) ? ' disabled' : ''}", data: { method: :post } do
+          safe_join([svg_logo, t('accounts.follow')])
+        end
+      end
+    elsif !(account.memorial? || account.moved?)
+      link_to account_remote_follow_path(account), class: 'button logo-button modal-button', target: '_new' do
+        safe_join([svg_logo, t('accounts.follow')])
+      end
+    end
+  end
+
+  def minimal_account_action_button(account)
+    if user_signed_in?
+      return if account.id == current_user.account_id
+
+      if current_account.following?(account) || current_account.requested?(account)
+        link_to account_unfollow_path(account), class: 'icon-button active', data: { method: :post }, title: t('accounts.unfollow') do
+          fa_icon('user-times fw')
+        end
+      elsif !(account.memorial? || account.moved?)
+        link_to account_follow_path(account), class: "icon-button#{account.blocking?(current_account) ? ' disabled' : ''}", data: { method: :post }, title: t('accounts.follow') do
+          fa_icon('user-plus fw')
+        end
+      end
+    elsif !(account.memorial? || account.moved?)
+      link_to account_remote_follow_path(account), class: 'icon-button modal-button', target: '_new', title: t('accounts.follow') do
+        fa_icon('user-plus fw')
+      end
+    end
+  end
+
+  def svg_logo
+    content_tag(:svg, tag(:use, 'xlink:href' => '#mastodon-svg-logo'), 'viewBox' => '0 0 216.4144 232.00976')
+  end
+
+  def svg_logo_full
+    content_tag(:svg, tag(:use, 'xlink:href' => '#mastodon-svg-logo-full'), 'viewBox' => '0 0 713.35878 175.8678')
+  end
+
+  def account_badge(account, all: false)
+    if account.bot?
+      content_tag(:div, content_tag(:div, t('accounts.roles.bot'), class: 'account-role bot'), class: 'roles')
+    elsif account.group?
+      content_tag(:div, content_tag(:div, t('accounts.roles.group'), class: 'account-role group'), class: 'roles')
+    elsif (Setting.show_staff_badge && account.user_staff?) || all
+      content_tag(:div, class: 'roles') do
+        if all && !account.user_staff?
+          content_tag(:div, t('admin.accounts.roles.user'), class: 'account-role')
+        elsif account.user_admin?
+          content_tag(:div, t('accounts.roles.admin'), class: 'account-role admin')
+        elsif account.user_moderator?
+          content_tag(:div, t('accounts.roles.moderator'), class: 'account-role moderator')
+        end
+      end
+    end
+  end
+
   def link_to_more(url)
     link_to t('statuses.show_more'), url, class: 'load-more load-gap'
   end
diff --git a/app/javascript/mastodon/features/account/components/header.js b/app/javascript/mastodon/features/account/components/header.js
index dbb567e85..8bd7f2db5 100644
--- a/app/javascript/mastodon/features/account/components/header.js
+++ b/app/javascript/mastodon/features/account/components/header.js
@@ -238,9 +238,18 @@ class Header extends ImmutablePureComponent {
     const content         = { __html: account.get('note_emojified') };
     const displayNameHtml = { __html: account.get('display_name_html') };
     const fields          = account.get('fields');
-    const badge           = account.get('bot') ? (<div className='account-role bot'><FormattedMessage id='account.badges.bot' defaultMessage='Bot' /></div>) : null;
     const acct            = account.get('acct').indexOf('@') === -1 && domain ? `${account.get('acct')}@${domain}` : account.get('acct');
 
+    let badge;
+
+    if (account.get('bot')) {
+      badge = (<div className='account-role bot'><FormattedMessage id='account.badges.bot' defaultMessage='Bot' /></div>);
+    } else if (account.get('group')) {
+      badge = (<div className='account-role group'><FormattedMessage id='account.badges.group' defaultMessage='Group' /></div>);
+    } else {
+      badge = null;
+    }
+
     return (
       <div className={classNames('account__header', { inactive: !!account.get('moved') })} ref={this.setRef}>
         <div className='account__header__image'>
diff --git a/app/lib/activitypub/activity.rb b/app/lib/activitypub/activity.rb
index cdd406043..0ca6b92a4 100644
--- a/app/lib/activitypub/activity.rb
+++ b/app/lib/activitypub/activity.rb
@@ -89,7 +89,7 @@ class ActivityPub::Activity
   def distribute(status)
     crawl_links(status)
 
-    notify_about_reblog(status) if reblog_of_local_account?(status)
+    notify_about_reblog(status) if reblog_of_local_account?(status) && !reblog_by_following_group_account?(status)
     notify_about_mentions(status)
 
     # Only continue if the status is supposed to have arrived in real-time.
@@ -105,6 +105,10 @@ class ActivityPub::Activity
     status.reblog? && status.reblog.account.local?
   end
 
+  def reblog_by_following_group_account?(status)
+    status.reblog? && status.account.group? && status.reblog.account.following?(status.account)
+  end
+
   def notify_about_reblog(status)
     NotifyService.new.call(status.reblog.account, status)
   end
diff --git a/app/lib/activitypub/tag_manager.rb b/app/lib/activitypub/tag_manager.rb
index 512272dbe..ed680d762 100644
--- a/app/lib/activitypub/tag_manager.rb
+++ b/app/lib/activitypub/tag_manager.rb
@@ -68,10 +68,19 @@ class ActivityPub::TagManager
       if status.account.silenced?
         # Only notify followers if the account is locally silenced
         account_ids = status.active_mentions.pluck(:account_id)
-        to = status.account.followers.where(id: account_ids).map { |account| uri_for(account) }
-        to.concat(FollowRequest.where(target_account_id: status.account_id, account_id: account_ids).map { |request| uri_for(request.account) })
+        to = status.account.followers.where(id: account_ids).each_with_object([]) do |account, result|
+          result << uri_for(account)
+          result << account.followers_url if account.group?
+        end
+        to.concat(FollowRequest.where(target_account_id: status.account_id, account_id: account_ids).each_with_object([]) do |request, result|
+          result << uri_for(request.account)
+          result << request.account.followers_url if request.account.group?
+        end)
       else
-        status.active_mentions.map { |mention| uri_for(mention.account) }
+        status.active_mentions.each_with_object([]) do |mention, result|
+          result << uri_for(mention.account)
+          result << mention.account.followers_url if mention.account.group?
+        end
       end
     end
   end
@@ -97,10 +106,19 @@ class ActivityPub::TagManager
       if status.account.silenced?
         # Only notify followers if the account is locally silenced
         account_ids = status.active_mentions.pluck(:account_id)
-        cc.concat(status.account.followers.where(id: account_ids).map { |account| uri_for(account) })
-        cc.concat(FollowRequest.where(target_account_id: status.account_id, account_id: account_ids).map { |request| uri_for(request.account) })
+        cc.concat(status.account.followers.where(id: account_ids).each_with_object([]) do |account, result|
+          result << uri_for(account)
+          result << account.followers_url if account.group?
+        end)
+        cc.concat(FollowRequest.where(target_account_id: status.account_id, account_id: account_ids).each_with_object([]) do |request, result|
+          result << uri_for(request.account)
+          result << request.account.followers_url if request.account.group?
+        end)
       else
-        cc.concat(status.active_mentions.map { |mention| uri_for(mention.account) })
+        cc.concat(status.active_mentions.each_with_object([]) do |mention, result|
+          result << uri_for(mention.account)
+          result << mention.account.followers_url if mention.account.group?
+        end)
       end
     end
 
diff --git a/app/models/account.rb b/app/models/account.rb
index d17782f78..884332e5a 100644
--- a/app/models/account.rb
+++ b/app/models/account.rb
@@ -93,6 +93,7 @@ class Account < ApplicationRecord
   scope :without_silenced, -> { where(silenced_at: nil) }
   scope :recent, -> { reorder(id: :desc) }
   scope :bots, -> { where(actor_type: %w(Application Service)) }
+  scope :groups, -> { where(actor_type: 'Group') }
   scope :alphabetic, -> { order(domain: :asc, username: :asc) }
   scope :by_domain_accounts, -> { group(:domain).select(:domain, 'COUNT(*) AS accounts_count').order('accounts_count desc') }
   scope :matches_username, ->(value) { where(arel_table[:username].matches("#{value}%")) }
@@ -153,6 +154,12 @@ class Account < ApplicationRecord
     self.actor_type = ActiveModel::Type::Boolean.new.cast(val) ? 'Service' : 'Person'
   end
 
+  def group?
+    actor_type == 'Group'
+  end
+
+  alias group group?
+
   def acct
     local? ? username : "#{username}@#{domain}"
   end
diff --git a/app/serializers/activitypub/actor_serializer.rb b/app/serializers/activitypub/actor_serializer.rb
index 17df85de3..aa64936a7 100644
--- a/app/serializers/activitypub/actor_serializer.rb
+++ b/app/serializers/activitypub/actor_serializer.rb
@@ -49,6 +49,8 @@ class ActivityPub::ActorSerializer < ActivityPub::Serializer
       'Application'
     elsif object.bot?
       'Service'
+    elsif object.group?
+      'Group'
     else
       'Person'
     end
diff --git a/app/serializers/rest/account_serializer.rb b/app/serializers/rest/account_serializer.rb
index 7e3041ae3..5fec75673 100644
--- a/app/serializers/rest/account_serializer.rb
+++ b/app/serializers/rest/account_serializer.rb
@@ -3,7 +3,7 @@
 class REST::AccountSerializer < ActiveModel::Serializer
   include RoutingHelper
 
-  attributes :id, :username, :acct, :display_name, :locked, :bot, :discoverable, :created_at,
+  attributes :id, :username, :acct, :display_name, :locked, :bot, :discoverable, :group, :created_at,
              :note, :url, :avatar, :avatar_static, :header, :header_static,
              :followers_count, :following_count, :statuses_count, :last_status_at
 
diff --git a/config/locales/en.yml b/config/locales/en.yml
index d498f6ce3..f6a14ad1a 100644
--- a/config/locales/en.yml
+++ b/config/locales/en.yml
@@ -78,6 +78,7 @@ en:
     roles:
       admin: Admin
       bot: Bot
+      group: Group
       moderator: Mod
     unavailable: Profile unavailable
     unfollow: Unfollow
-- 
cgit