From 339ce1c4e90605b736745b1f04493a247b2627ec Mon Sep 17 00:00:00 2001
From: Eugen Rochko <eugen@zeonfederated.com>
Date: Sun, 8 Mar 2020 15:17:39 +0100
Subject: Add specific rate limits for posting and following (#13172)

---
 app/services/follow_service.rb      | 78 ++++++++++++++++++++++---------------
 app/services/post_status_service.rb | 11 ++++--
 app/services/reblog_service.rb      | 16 ++++++--
 3 files changed, 66 insertions(+), 39 deletions(-)

(limited to 'app/services')

diff --git a/app/services/follow_service.rb b/app/services/follow_service.rb
index 4d19002c4..311ae7fa6 100644
--- a/app/services/follow_service.rb
+++ b/app/services/follow_service.rb
@@ -7,54 +7,68 @@ class FollowService < BaseService
   # Follow a remote user, notify remote user about the follow
   # @param [Account] source_account From which to follow
   # @param [String, Account] uri User URI to follow in the form of username@domain (or account record)
-  # @param [true, false, nil] reblogs Whether or not to show reblogs, defaults to true
-  def call(source_account, target_account, reblogs: nil, bypass_locked: false)
-    reblogs = true if reblogs.nil?
-    target_account = ResolveAccountService.new.call(target_account, skip_webfinger: true)
-
-    raise ActiveRecord::RecordNotFound if target_account.nil? || target_account.id == source_account.id || target_account.suspended?
-    raise Mastodon::NotPermittedError  if target_account.blocking?(source_account) || source_account.blocking?(target_account) || target_account.moved? || (!target_account.local? && target_account.ostatus?) || source_account.domain_blocking?(target_account.domain)
-
-    if source_account.following?(target_account)
-      # We're already following this account, but we'll call follow! again to
-      # make sure the reblogs status is set correctly.
-      return source_account.follow!(target_account, reblogs: reblogs)
-    elsif source_account.requested?(target_account)
-      # This isn't managed by a method in AccountInteractions, so we modify it
-      # ourselves if necessary.
-      req = source_account.follow_requests.find_by(target_account: target_account)
-      req.update!(show_reblogs: reblogs)
-      return req
+  # @param [Hash] options
+  # @option [Boolean] :reblogs Whether or not to show reblogs, defaults to true
+  # @option [Boolean] :bypass_locked
+  # @option [Boolean] :with_rate_limit
+  def call(source_account, target_account, options = {})
+    @source_account = source_account
+    @target_account = ResolveAccountService.new.call(target_account, skip_webfinger: true)
+    @options        = { reblogs: true, bypass_locked: false, with_rate_limit: false }.merge(options)
+
+    raise ActiveRecord::RecordNotFound if following_not_possible?
+    raise Mastodon::NotPermittedError  if following_not_allowed?
+
+    if @source_account.following?(@target_account)
+      return change_follow_options!
+    elsif @source_account.requested?(@target_account)
+      return change_follow_request_options!
     end
 
     ActivityTracker.increment('activity:interactions')
 
-    if (target_account.locked? && !bypass_locked) || source_account.silenced? || target_account.activitypub?
-      request_follow(source_account, target_account, reblogs: reblogs)
-    elsif target_account.local?
-      direct_follow(source_account, target_account, reblogs: reblogs)
+    if (@target_account.locked? && !@options[:bypass_locked]) || @source_account.silenced? || @target_account.activitypub?
+      request_follow!
+    elsif @target_account.local?
+      direct_follow!
     end
   end
 
   private
 
-  def request_follow(source_account, target_account, reblogs: true)
-    follow_request = FollowRequest.create!(account: source_account, target_account: target_account, show_reblogs: reblogs)
+  def following_not_possible?
+    @target_account.nil? || @target_account.id == @source_account.id || @target_account.suspended?
+  end
+
+  def following_not_allowed?
+    @target_account.blocking?(@source_account) || @source_account.blocking?(@target_account) || @target_account.moved? || (!@target_account.local? && @target_account.ostatus?) || @source_account.domain_blocking?(@target_account.domain)
+  end
+
+  def change_follow_options!
+    @source_account.follow!(@target_account, reblogs: @options[:reblogs])
+  end
+
+  def change_follow_request_options!
+    @source_account.request_follow!(@target_account, reblogs: @options[:reblogs])
+  end
+
+  def request_follow!
+    follow_request = @source_account.request_follow!(@target_account, reblogs: @options[:reblogs], rate_limit: @options[:with_rate_limit])
 
-    if target_account.local?
-      LocalNotificationWorker.perform_async(target_account.id, follow_request.id, follow_request.class.name)
-    elsif target_account.activitypub?
-      ActivityPub::DeliveryWorker.perform_async(build_json(follow_request), source_account.id, target_account.inbox_url)
+    if @target_account.local?
+      LocalNotificationWorker.perform_async(@target_account.id, follow_request.id, follow_request.class.name)
+    elsif @target_account.activitypub?
+      ActivityPub::DeliveryWorker.perform_async(build_json(follow_request), @source_account.id, @target_account.inbox_url)
     end
 
     follow_request
   end
 
-  def direct_follow(source_account, target_account, reblogs: true)
-    follow = source_account.follow!(target_account, reblogs: reblogs)
+  def direct_follow!
+    follow = @source_account.follow!(@target_account, reblogs: @options[:reblogs], rate_limit: @options[:with_rate_limit])
 
-    LocalNotificationWorker.perform_async(target_account.id, follow.id, follow.class.name)
-    MergeWorker.perform_async(target_account.id, source_account.id)
+    LocalNotificationWorker.perform_async(@target_account.id, follow.id, follow.class.name)
+    MergeWorker.perform_async(@target_account.id, @source_account.id)
 
     follow
   end
diff --git a/app/services/post_status_service.rb b/app/services/post_status_service.rb
index a0a650d62..1bbd625b4 100644
--- a/app/services/post_status_service.rb
+++ b/app/services/post_status_service.rb
@@ -19,6 +19,7 @@ class PostStatusService < BaseService
   # @option [Enumerable] :media_ids Optional array of media IDs to attach
   # @option [Doorkeeper::Application] :application
   # @option [String] :idempotency Optional idempotency key
+  # @option [Boolean] :with_rate_limit
   # @return [Status]
   def call(account, options = {})
     @account     = account
@@ -160,6 +161,7 @@ class PostStatusService < BaseService
       visibility: @visibility,
       language: language_from_option(@options[:language]) || @account.user&.setting_default_language&.presence || LanguageDetector.instance.detect(@text, @account),
       application: @options[:application],
+      rate_limit: @options[:with_rate_limit],
     }.compact
   end
 
@@ -179,10 +181,11 @@ class PostStatusService < BaseService
 
   def scheduled_options
     @options.tap do |options_hash|
-      options_hash[:in_reply_to_id] = options_hash.delete(:thread)&.id
-      options_hash[:application_id] = options_hash.delete(:application)&.id
-      options_hash[:scheduled_at]   = nil
-      options_hash[:idempotency]    = nil
+      options_hash[:in_reply_to_id]  = options_hash.delete(:thread)&.id
+      options_hash[:application_id]  = options_hash.delete(:application)&.id
+      options_hash[:scheduled_at]    = nil
+      options_hash[:idempotency]     = nil
+      options_hash[:with_rate_limit] = false
     end
   end
 end
diff --git a/app/services/reblog_service.rb b/app/services/reblog_service.rb
index 3bb460fca..4b5ae9492 100644
--- a/app/services/reblog_service.rb
+++ b/app/services/reblog_service.rb
@@ -8,6 +8,8 @@ class ReblogService < BaseService
   # @param [Account] account Account to reblog from
   # @param [Status] reblogged_status Status to be reblogged
   # @param [Hash] options
+  # @option [String]  :visibility
+  # @option [Boolean] :with_rate_limit
   # @return [Status]
   def call(account, reblogged_status, options = {})
     reblogged_status = reblogged_status.reblog if reblogged_status.reblog?
@@ -18,9 +20,15 @@ class ReblogService < BaseService
 
     return reblog unless reblog.nil?
 
-    visibility = options[:visibility] || account.user&.setting_default_privacy
-    visibility = reblogged_status.visibility if reblogged_status.hidden?
-    reblog = account.statuses.create!(reblog: reblogged_status, text: '', visibility: visibility)
+    visibility = begin
+      if reblogged_status.hidden?
+        reblogged_status.visibility
+      else
+        options[:visibility] || account.user&.setting_default_privacy
+      end
+    end
+
+    reblog = account.statuses.create!(reblog: reblogged_status, text: '', visibility: visibility, rate_limit: options[:with_rate_limit])
 
     DistributionWorker.perform_async(reblog.id)
     ActivityPub::DistributionWorker.perform_async(reblog.id)
@@ -45,7 +53,9 @@ class ReblogService < BaseService
 
   def bump_potential_friendship(account, reblog)
     ActivityTracker.increment('activity:interactions')
+
     return if account.following?(reblog.reblog.account_id)
+
     PotentialFriendshipTracker.record(account.id, reblog.reblog.account_id, :reblog)
   end
 
-- 
cgit 


From 6185bff4b3c2fec7b84aee8a9d6747d42ad865c0 Mon Sep 17 00:00:00 2001
From: ThibG <thib@sitedethib.com>
Date: Sun, 8 Mar 2020 15:42:20 +0100
Subject: Fix error when searching for URLs that contain the mention syntax
 (#13151)

Fixes #13150
---
 app/services/account_search_service.rb | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

(limited to 'app/services')

diff --git a/app/services/account_search_service.rb b/app/services/account_search_service.rb
index d217dabb3..493813aab 100644
--- a/app/services/account_search_service.rb
+++ b/app/services/account_search_service.rb
@@ -171,7 +171,7 @@ class AccountSearchService < BaseService
   end
 
   def username_complete?
-    query.include?('@') && "@#{query}" =~ Account::MENTION_RE
+    query.include?('@') && "@#{query}" =~ /\A#{Account::MENTION_RE}\Z/
   end
 
   def likely_acct?
-- 
cgit 


From 5284e29e2f0f2fd637c661b7063a3058e45d8f25 Mon Sep 17 00:00:00 2001
From: ThibG <thib@sitedethib.com>
Date: Sun, 8 Mar 2020 16:11:49 +0100
Subject: Fix public posts from silenced accounts not being changed to unlisted
 visibility (#13096)

---
 app/services/post_status_service.rb | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

(limited to 'app/services')

diff --git a/app/services/post_status_service.rb b/app/services/post_status_service.rb
index 1bbd625b4..2301621ab 100644
--- a/app/services/post_status_service.rb
+++ b/app/services/post_status_service.rb
@@ -50,7 +50,7 @@ class PostStatusService < BaseService
   def preprocess_attributes!
     @text         = @options.delete(:spoiler_text) if @text.blank? && @options[:spoiler_text].present?
     @visibility   = @options[:visibility] || @account.user&.setting_default_privacy
-    @visibility   = :unlisted if @visibility == :public && @account.silenced?
+    @visibility   = :unlisted if @visibility&.to_sym == :public && @account.silenced?
     @scheduled_at = @options[:scheduled_at]&.to_datetime
     @scheduled_at = nil if scheduled_in_the_past?
   rescue ArgumentError
-- 
cgit 


From 9660aa4543deff41c60d131e081137f84e771499 Mon Sep 17 00:00:00 2001
From: Eugen Rochko <eugen@zeonfederated.com>
Date: Sun, 8 Mar 2020 23:56:18 +0100
Subject: Change local media attachments to perform heavy processing
 asynchronously (#13210)

Fix #9106
---
 app/controllers/api/v1/media_controller.rb         | 29 ++++++++++++++----
 app/controllers/api/v2/media_controller.rb         | 12 ++++++++
 app/javascript/mastodon/actions/compose.js         | 23 +++++++++++++--
 app/models/concerns/attachmentable.rb              |  2 +-
 app/models/media_attachment.rb                     | 26 +++++++++++++++--
 .../rest/media_attachment_serializer.rb            |  4 ++-
 app/services/post_status_service.rb                |  1 +
 app/workers/backup_worker.rb                       |  8 +++--
 app/workers/post_process_media_worker.rb           | 34 ++++++++++++++++++++++
 config/application.rb                              |  1 +
 config/locales/en.yml                              |  1 +
 config/routes.rb                                   |  3 +-
 ...06035625_add_processing_to_media_attachments.rb |  5 ++++
 db/schema.rb                                       |  3 +-
 lib/paperclip/attachment_extensions.rb             | 30 +++++++++++++++++++
 15 files changed, 165 insertions(+), 17 deletions(-)
 create mode 100644 app/controllers/api/v2/media_controller.rb
 create mode 100644 app/workers/post_process_media_worker.rb
 create mode 100644 db/migrate/20200306035625_add_processing_to_media_attachments.rb
 create mode 100644 lib/paperclip/attachment_extensions.rb

(limited to 'app/services')

diff --git a/app/controllers/api/v1/media_controller.rb b/app/controllers/api/v1/media_controller.rb
index d87d7b946..0bb3d0d27 100644
--- a/app/controllers/api/v1/media_controller.rb
+++ b/app/controllers/api/v1/media_controller.rb
@@ -3,25 +3,42 @@
 class Api::V1::MediaController < Api::BaseController
   before_action -> { doorkeeper_authorize! :write, :'write:media' }
   before_action :require_user!
+  before_action :set_media_attachment, except: [:create]
+  before_action :check_processing, except: [:create]
 
   def create
-    @media = current_account.media_attachments.create!(media_params)
-    render json: @media, serializer: REST::MediaAttachmentSerializer
+    @media_attachment = current_account.media_attachments.create!(media_attachment_params)
+    render json: @media_attachment, serializer: REST::MediaAttachmentSerializer
   rescue Paperclip::Errors::NotIdentifiedByImageMagickError
     render json: file_type_error, status: 422
   rescue Paperclip::Error
     render json: processing_error, status: 500
   end
 
+  def show
+    render json: @media_attachment, serializer: REST::MediaAttachmentSerializer, status: status_code_for_media_attachment
+  end
+
   def update
-    @media = current_account.media_attachments.where(status_id: nil).find(params[:id])
-    @media.update!(media_params)
-    render json: @media, serializer: REST::MediaAttachmentSerializer
+    @media_attachment.update!(media_attachment_params)
+    render json: @media_attachment, serializer: REST::MediaAttachmentSerializer, status: status_code_for_media_attachment
   end
 
   private
 
-  def media_params
+  def status_code_for_media_attachment
+    @media_attachment.not_processed? ? 206 : 200
+  end
+
+  def set_media_attachment
+    @media_attachment = current_account.media_attachments.unattached.find(params[:id])
+  end
+
+  def check_processing
+    render json: processing_error, status: 422 if @media_attachment.processing_failed?
+  end
+
+  def media_attachment_params
     params.permit(:file, :description, :focus)
   end
 
diff --git a/app/controllers/api/v2/media_controller.rb b/app/controllers/api/v2/media_controller.rb
new file mode 100644
index 000000000..0c1baf01d
--- /dev/null
+++ b/app/controllers/api/v2/media_controller.rb
@@ -0,0 +1,12 @@
+# frozen_string_literal: true
+
+class Api::V2::MediaController < Api::V1::MediaController
+  def create
+    @media_attachment = current_account.media_attachments.create!({ delay_processing: true }.merge(media_attachment_params))
+    render json: @media_attachment, serializer: REST::MediaAttachmentSerializer, status: 202
+  rescue Paperclip::Errors::NotIdentifiedByImageMagickError
+    render json: file_type_error, status: 422
+  rescue Paperclip::Error
+    render json: processing_error, status: 500
+  end
+end
diff --git a/app/javascript/mastodon/actions/compose.js b/app/javascript/mastodon/actions/compose.js
index c3c6ff1a1..68ef8fbd5 100644
--- a/app/javascript/mastodon/actions/compose.js
+++ b/app/javascript/mastodon/actions/compose.js
@@ -230,12 +230,31 @@ export function uploadCompose(files) {
         // Account for disparity in size of original image and resized data
         total += file.size - f.size;
 
-        return api(getState).post('/api/v1/media', data, {
+        return api(getState).post('/api/v2/media', data, {
           onUploadProgress: function({ loaded }){
             progress[i] = loaded;
             dispatch(uploadComposeProgress(progress.reduce((a, v) => a + v, 0), total));
           },
-        }).then(({ data }) => dispatch(uploadComposeSuccess(data, f)));
+        }).then(({ status, data }) => {
+          // If server-side processing of the media attachment has not completed yet,
+          // poll the server until it is, before showing the media attachment as uploaded
+
+          if (status === 200) {
+            dispatch(uploadComposeSuccess(data, f));
+          } else if (status === 202) {
+            const poll = () => {
+              api(getState).get(`/api/v1/media/${data.id}`).then(response => {
+                if (response.status === 200) {
+                  dispatch(uploadComposeSuccess(data, f));
+                } else if (response.status === 206) {
+                  setTimeout(() => poll(), 1000);
+                }
+              }).catch(error => dispatch(uploadComposeFail(error)));
+            };
+
+            poll();
+          }
+        });
       }).catch(error => dispatch(uploadComposeFail(error)));
     };
   };
diff --git a/app/models/concerns/attachmentable.rb b/app/models/concerns/attachmentable.rb
index 43ff8ac12..18b872c1e 100644
--- a/app/models/concerns/attachmentable.rb
+++ b/app/models/concerns/attachmentable.rb
@@ -74,7 +74,7 @@ module Attachmentable
     self.class.attachment_definitions.each_key do |attachment_name|
       attachment = send(attachment_name)
 
-      next if attachment.blank? || attachment.queued_for_write[:original].blank?
+      next if attachment.blank? || attachment.queued_for_write[:original].blank? || attachment.options[:preserve_files]
 
       attachment.instance_write :file_name, SecureRandom.hex(8) + File.extname(attachment.instance_read(:file_name))
     end
diff --git a/app/models/media_attachment.rb b/app/models/media_attachment.rb
index 1e36625ca..3fd4f231c 100644
--- a/app/models/media_attachment.rb
+++ b/app/models/media_attachment.rb
@@ -19,12 +19,14 @@
 #  description         :text
 #  scheduled_status_id :bigint(8)
 #  blurhash            :string
+#  processing          :integer
 #
 
 class MediaAttachment < ApplicationRecord
   self.inheritance_column = nil
 
   enum type: [:image, :gifv, :video, :unknown, :audio]
+  enum processing: [:queued, :in_progress, :complete, :failed], _prefix: true
 
   MAX_DESCRIPTION_LENGTH = 1_500
 
@@ -156,6 +158,10 @@ class MediaAttachment < ApplicationRecord
     remote_url.blank?
   end
 
+  def not_processed?
+    processing.present? && !processing_complete?
+  end
+
   def needs_redownload?
     file.blank? && remote_url.present?
   end
@@ -203,10 +209,18 @@ class MediaAttachment < ApplicationRecord
     "#{x},#{y}"
   end
 
+  attr_writer :delay_processing
+
+  def delay_processing?
+    @delay_processing
+  end
+
+  after_commit :enqueue_processing, on: :create
   after_commit :reset_parent_cache, on: :update
 
   before_create :prepare_description, unless: :local?
   before_create :set_shortcode
+  before_create :set_processing
 
   before_post_process :set_type_and_extension
 
@@ -277,6 +291,10 @@ class MediaAttachment < ApplicationRecord
     end
   end
 
+  def set_processing
+    self.processing = delay_processing? ? :queued : :complete
+  end
+
   def set_meta
     meta = populate_meta
 
@@ -322,9 +340,11 @@ class MediaAttachment < ApplicationRecord
     }.compact
   end
 
-  def reset_parent_cache
-    return if status_id.nil?
+  def enqueue_processing
+    PostProcessMediaWorker.perform_async(id) if delay_processing?
+  end
 
-    Rails.cache.delete("statuses/#{status_id}")
+  def reset_parent_cache
+    Rails.cache.delete("statuses/#{status_id}") if status_id.present?
   end
 end
diff --git a/app/serializers/rest/media_attachment_serializer.rb b/app/serializers/rest/media_attachment_serializer.rb
index 1b3498ea4..cc10e3001 100644
--- a/app/serializers/rest/media_attachment_serializer.rb
+++ b/app/serializers/rest/media_attachment_serializer.rb
@@ -12,7 +12,9 @@ class REST::MediaAttachmentSerializer < ActiveModel::Serializer
   end
 
   def url
-    if object.needs_redownload?
+    if object.not_processed?
+      nil
+    elsif object.needs_redownload?
       media_proxy_url(object.id, :original)
     else
       full_asset_url(object.file.url(:original))
diff --git a/app/services/post_status_service.rb b/app/services/post_status_service.rb
index 2301621ab..c61b3baa2 100644
--- a/app/services/post_status_service.rb
+++ b/app/services/post_status_service.rb
@@ -101,6 +101,7 @@ class PostStatusService < BaseService
     @media = @account.media_attachments.where(status_id: nil).where(id: @options[:media_ids].take(4).map(&:to_i))
 
     raise Mastodon::ValidationError, I18n.t('media_attachments.validations.images_and_video') if @media.size > 1 && @media.find(&:audio_or_video?)
+    raise Mastodon::ValidationError, I18n.t('media_attachments.validations.not_ready') if @media.any?(&:not_processed?)
   end
 
   def language_from_option(str)
diff --git a/app/workers/backup_worker.rb b/app/workers/backup_worker.rb
index e4c609d70..7b0b52844 100644
--- a/app/workers/backup_worker.rb
+++ b/app/workers/backup_worker.rb
@@ -9,8 +9,12 @@ class BackupWorker
     backup_id = msg['args'].first
 
     ActiveRecord::Base.connection_pool.with_connection do
-      backup = Backup.find(backup_id)
-      backup&.destroy
+      begin
+        backup = Backup.find(backup_id)
+        backup.destroy
+      rescue ActiveRecord::RecordNotFound
+        true
+      end
     end
   end
 
diff --git a/app/workers/post_process_media_worker.rb b/app/workers/post_process_media_worker.rb
new file mode 100644
index 000000000..d3ebda194
--- /dev/null
+++ b/app/workers/post_process_media_worker.rb
@@ -0,0 +1,34 @@
+# frozen_string_literal: true
+
+class PostProcessMediaWorker
+  include Sidekiq::Worker
+
+  sidekiq_options retry: 1, dead: false
+
+  sidekiq_retries_exhausted do |msg|
+    media_attachment_id = msg['args'].first
+
+    ActiveRecord::Base.connection_pool.with_connection do
+      begin
+        media_attachment = MediaAttachment.find(media_attachment_id)
+        media_attachment.processing = :failed
+        media_attachment.save
+      rescue ActiveRecord::RecordNotFound
+        true
+      end
+    end
+
+    Sidekiq.logger.error("Processing media attachment #{media_attachment_id} failed with #{msg['error_message']}")
+  end
+
+  def perform(media_attachment_id)
+    media_attachment = MediaAttachment.find(media_attachment_id)
+    media_attachment.processing = :in_progress
+    media_attachment.save
+    media_attachment.file.reprocess_original!
+    media_attachment.processing = :complete
+    media_attachment.save
+  rescue ActiveRecord::RecordNotFound
+    true
+  end
+end
diff --git a/config/application.rb b/config/application.rb
index 1baa166ce..cd599eefc 100644
--- a/config/application.rb
+++ b/config/application.rb
@@ -7,6 +7,7 @@ require 'rails/all'
 Bundler.require(*Rails.groups)
 
 require_relative '../app/lib/exceptions'
+require_relative '../lib/paperclip/attachment_extensions'
 require_relative '../lib/paperclip/lazy_thumbnail'
 require_relative '../lib/paperclip/gif_transcoder'
 require_relative '../lib/paperclip/video_transcoder'
diff --git a/config/locales/en.yml b/config/locales/en.yml
index 8440b471c..a031f9e9d 100644
--- a/config/locales/en.yml
+++ b/config/locales/en.yml
@@ -853,6 +853,7 @@ en:
   media_attachments:
     validations:
       images_and_video: Cannot attach a video to a status that already contains images
+      not_ready: Cannot attach files that have not finished processing. Try again in a moment!
       too_many: Cannot attach more than 4 files
   migrations:
     acct: Moved to
diff --git a/config/routes.rb b/config/routes.rb
index 2de31e2db..89d446d10 100644
--- a/config/routes.rb
+++ b/config/routes.rb
@@ -343,7 +343,7 @@ Rails.application.routes.draw do
         end
       end
 
-      resources :media,        only: [:create, :update]
+      resources :media,        only: [:create, :update, :show]
       resources :blocks,       only: [:index]
       resources :mutes,        only: [:index]
       resources :favourites,   only: [:index]
@@ -455,6 +455,7 @@ Rails.application.routes.draw do
     end
 
     namespace :v2 do
+      resources :media, only: [:create]
       get '/search', to: 'search#index', as: :search
     end
 
diff --git a/db/migrate/20200306035625_add_processing_to_media_attachments.rb b/db/migrate/20200306035625_add_processing_to_media_attachments.rb
new file mode 100644
index 000000000..131ffa52a
--- /dev/null
+++ b/db/migrate/20200306035625_add_processing_to_media_attachments.rb
@@ -0,0 +1,5 @@
+class AddProcessingToMediaAttachments < ActiveRecord::Migration[5.2]
+  def change
+    add_column :media_attachments, :processing, :integer
+  end
+end
diff --git a/db/schema.rb b/db/schema.rb
index b09ee0e76..ddb9a5d8c 100644
--- a/db/schema.rb
+++ b/db/schema.rb
@@ -10,7 +10,7 @@
 #
 # It's strongly recommended that you check this file into your version control system.
 
-ActiveRecord::Schema.define(version: 2020_01_26_203551) do
+ActiveRecord::Schema.define(version: 2020_03_06_035625) do
 
   # These are extensions that must be enabled in order to support this database
   enable_extension "plpgsql"
@@ -459,6 +459,7 @@ ActiveRecord::Schema.define(version: 2020_01_26_203551) do
     t.text "description"
     t.bigint "scheduled_status_id"
     t.string "blurhash"
+    t.integer "processing"
     t.index ["account_id"], name: "index_media_attachments_on_account_id"
     t.index ["scheduled_status_id"], name: "index_media_attachments_on_scheduled_status_id"
     t.index ["shortcode"], name: "index_media_attachments_on_shortcode", unique: true
diff --git a/lib/paperclip/attachment_extensions.rb b/lib/paperclip/attachment_extensions.rb
new file mode 100644
index 000000000..3b308af5f
--- /dev/null
+++ b/lib/paperclip/attachment_extensions.rb
@@ -0,0 +1,30 @@
+# frozen_string_literal: true
+
+module Paperclip
+  module AttachmentExtensions
+    # We overwrite this method to support delayed processing in
+    # Sidekiq. Since we process the original file to reduce disk
+    # usage, and we still want to generate thumbnails straight
+    # away, it's the only style we need to exclude
+    def process_style?(style_name, style_args)
+      if style_name == :original && instance.respond_to?(:delay_processing?) && instance.delay_processing?
+        false
+      else
+        style_args.empty? || style_args.include?(style_name)
+      end
+    end
+
+    def reprocess_original!
+      old_original_path = path(:original)
+      reprocess!(:original)
+      new_original_path = path(:original)
+
+      if new_original_path != old_original_path
+        @queued_for_delete << old_original_path
+        flush_deletes
+      end
+    end
+  end
+end
+
+Paperclip::Attachment.prepend(Paperclip::AttachmentExtensions)
-- 
cgit 


From b154428e14861f5cdc7ba6e5f8e582dbf7d0a1c0 Mon Sep 17 00:00:00 2001
From: ThibG <thib@sitedethib.com>
Date: Mon, 9 Mar 2020 00:10:29 +0100
Subject: Add federation support for the "hide network" preference (#11673)

* Change ActivityPub follower/following collections to not link first page

* Add support for hiding followers and following of remote users

* Switch to using a single `hide_collections` column

* Address code style remarks
---
 .../v1/accounts/follower_accounts_controller.rb    |  2 +-
 .../v1/accounts/following_accounts_controller.rb   |  2 +-
 app/controllers/follower_accounts_controller.rb    | 11 +++++++++-
 app/controllers/following_accounts_controller.rb   | 11 +++++++++-
 app/models/account.rb                              |  9 ++++++++
 .../activitypub/process_account_service.rb         | 25 ++++++++++++++++------
 ...91212163405_add_hide_collections_to_accounts.rb |  5 +++++
 db/schema.rb                                       |  1 +
 8 files changed, 55 insertions(+), 11 deletions(-)
 create mode 100644 db/migrate/20191212163405_add_hide_collections_to_accounts.rb

(limited to 'app/services')

diff --git a/app/controllers/api/v1/accounts/follower_accounts_controller.rb b/app/controllers/api/v1/accounts/follower_accounts_controller.rb
index 850702cca..1daa1ed0d 100644
--- a/app/controllers/api/v1/accounts/follower_accounts_controller.rb
+++ b/app/controllers/api/v1/accounts/follower_accounts_controller.rb
@@ -25,7 +25,7 @@ class Api::V1::Accounts::FollowerAccountsController < Api::BaseController
   end
 
   def hide_results?
-    (@account.user_hides_network? && current_account&.id != @account.id) || (current_account && @account.blocking?(current_account))
+    (@account.hides_followers? && current_account&.id != @account.id) || (current_account && @account.blocking?(current_account))
   end
 
   def default_accounts
diff --git a/app/controllers/api/v1/accounts/following_accounts_controller.rb b/app/controllers/api/v1/accounts/following_accounts_controller.rb
index 830dcd8a1..6fc23cf75 100644
--- a/app/controllers/api/v1/accounts/following_accounts_controller.rb
+++ b/app/controllers/api/v1/accounts/following_accounts_controller.rb
@@ -25,7 +25,7 @@ class Api::V1::Accounts::FollowingAccountsController < Api::BaseController
   end
 
   def hide_results?
-    (@account.user_hides_network? && current_account&.id != @account.id) || (current_account && @account.blocking?(current_account))
+    (@account.hides_following? && current_account&.id != @account.id) || (current_account && @account.blocking?(current_account))
   end
 
   def default_accounts
diff --git a/app/controllers/follower_accounts_controller.rb b/app/controllers/follower_accounts_controller.rb
index 7103749ad..14e22dd1e 100644
--- a/app/controllers/follower_accounts_controller.rb
+++ b/app/controllers/follower_accounts_controller.rb
@@ -28,7 +28,8 @@ class FollowerAccountsController < ApplicationController
         render json: collection_presenter,
                serializer: ActivityPub::CollectionSerializer,
                adapter: ActivityPub::Adapter,
-               content_type: 'application/activity+json'
+               content_type: 'application/activity+json',
+               fields: restrict_fields_to
       end
     end
   end
@@ -71,4 +72,12 @@ class FollowerAccountsController < ApplicationController
       )
     end
   end
+
+  def restrict_fields_to
+    if page_requested? || !@account.user_hides_network?
+      # Return all fields
+    else
+      %i(id type totalItems)
+    end
+  end
 end
diff --git a/app/controllers/following_accounts_controller.rb b/app/controllers/following_accounts_controller.rb
index 6c8fb84d8..95849ffb9 100644
--- a/app/controllers/following_accounts_controller.rb
+++ b/app/controllers/following_accounts_controller.rb
@@ -28,7 +28,8 @@ class FollowingAccountsController < ApplicationController
         render json: collection_presenter,
                serializer: ActivityPub::CollectionSerializer,
                adapter: ActivityPub::Adapter,
-               content_type: 'application/activity+json'
+               content_type: 'application/activity+json',
+               fields: restrict_fields_to
       end
     end
   end
@@ -71,4 +72,12 @@ class FollowingAccountsController < ApplicationController
       )
     end
   end
+
+  def restrict_fields_to
+    if page_requested? || !@account.user_hides_network?
+      # Return all fields
+    else
+      %i(id type totalItems)
+    end
+  end
 end
diff --git a/app/models/account.rb b/app/models/account.rb
index a1b4a065b..6aceb809c 100644
--- a/app/models/account.rb
+++ b/app/models/account.rb
@@ -46,6 +46,7 @@
 #  silenced_at             :datetime
 #  suspended_at            :datetime
 #  trust_level             :integer
+#  hide_collections        :boolean
 #
 
 class Account < ApplicationRecord
@@ -323,6 +324,14 @@ class Account < ApplicationRecord
     save!
   end
 
+  def hides_followers?
+    hide_collections? || user_hides_network?
+  end
+
+  def hides_following?
+    hide_collections? || user_hides_network?
+  end
+
   def object_type
     :person
   end
diff --git a/app/services/activitypub/process_account_service.rb b/app/services/activitypub/process_account_service.rb
index d5ede0388..7b4c53d50 100644
--- a/app/services/activitypub/process_account_service.rb
+++ b/app/services/activitypub/process_account_service.rb
@@ -94,6 +94,7 @@ class ActivityPub::ProcessAccountService < BaseService
     @account.statuses_count    = outbox_total_items    if outbox_total_items.present?
     @account.following_count   = following_total_items if following_total_items.present?
     @account.followers_count   = followers_total_items if followers_total_items.present?
+    @account.hide_collections  = following_private? || followers_private?
     @account.moved_to_account  = @json['movedTo'].present? ? moved_account : nil
   end
 
@@ -166,26 +167,36 @@ class ActivityPub::ProcessAccountService < BaseService
   end
 
   def outbox_total_items
-    collection_total_items('outbox')
+    collection_info('outbox').first
   end
 
   def following_total_items
-    collection_total_items('following')
+    collection_info('following').first
   end
 
   def followers_total_items
-    collection_total_items('followers')
+    collection_info('followers').first
   end
 
-  def collection_total_items(type)
-    return if @json[type].blank?
+  def following_private?
+    !collection_info('following').last
+  end
+
+  def followers_private?
+    !collection_info('followers').last
+  end
+
+  def collection_info(type)
+    return [nil, nil] if @json[type].blank?
     return @collections[type] if @collections.key?(type)
 
     collection = fetch_resource_without_id_validation(@json[type])
 
-    @collections[type] = collection.is_a?(Hash) && collection['totalItems'].present? && collection['totalItems'].is_a?(Numeric) ? collection['totalItems'] : nil
+    total_items = collection.is_a?(Hash) && collection['totalItems'].present? && collection['totalItems'].is_a?(Numeric) ? collection['totalItems'] : nil
+    has_first_page = collection.is_a?(Hash) && collection['first'].present?
+    @collections[type] = [total_items, has_first_page]
   rescue HTTP::Error, OpenSSL::SSL::SSLError
-    @collections[type] = nil
+    @collections[type] = [nil, nil]
   end
 
   def moved_account
diff --git a/db/migrate/20191212163405_add_hide_collections_to_accounts.rb b/db/migrate/20191212163405_add_hide_collections_to_accounts.rb
new file mode 100644
index 000000000..fa99b32e5
--- /dev/null
+++ b/db/migrate/20191212163405_add_hide_collections_to_accounts.rb
@@ -0,0 +1,5 @@
+class AddHideCollectionsToAccounts < ActiveRecord::Migration[5.2]
+  def change
+    add_column :accounts, :hide_collections, :boolean
+  end
+end
diff --git a/db/schema.rb b/db/schema.rb
index ddb9a5d8c..a851e01fc 100644
--- a/db/schema.rb
+++ b/db/schema.rb
@@ -170,6 +170,7 @@ ActiveRecord::Schema.define(version: 2020_03_06_035625) do
     t.datetime "silenced_at"
     t.datetime "suspended_at"
     t.integer "trust_level"
+    t.boolean "hide_collections"
     t.index "(((setweight(to_tsvector('simple'::regconfig, (display_name)::text), 'A'::\"char\") || setweight(to_tsvector('simple'::regconfig, (username)::text), 'B'::\"char\")) || setweight(to_tsvector('simple'::regconfig, (COALESCE(domain, ''::character varying))::text), 'C'::\"char\")))", name: "search_index", using: :gin
     t.index "lower((username)::text), lower((domain)::text)", name: "index_accounts_on_username_and_domain_lower", unique: true
     t.index ["moved_to_account_id"], name: "index_accounts_on_moved_to_account_id"
-- 
cgit 


From cb12a2cdd3fbe2c07a556610596a2de5446b1f50 Mon Sep 17 00:00:00 2001
From: ThibG <thib@sitedethib.com>
Date: Thu, 12 Mar 2020 23:06:43 +0100
Subject: Fix some timeouts when searching URLs by limiting some database
 queries (#13253)

Only look up private toots from database if the request failed because of 401,
403 or 404 errors, as those may indicate a private toot, rather than something
that isn't a toot or cannot be processed.
---
 app/services/fetch_resource_service.rb |  3 +++
 app/services/resolve_url_service.rb    | 10 ++++++++--
 2 files changed, 11 insertions(+), 2 deletions(-)

(limited to 'app/services')

diff --git a/app/services/fetch_resource_service.rb b/app/services/fetch_resource_service.rb
index abe7766d4..880cdde92 100644
--- a/app/services/fetch_resource_service.rb
+++ b/app/services/fetch_resource_service.rb
@@ -5,6 +5,8 @@ class FetchResourceService < BaseService
 
   ACCEPT_HEADER = 'application/activity+json, application/ld+json; profile="https://www.w3.org/ns/activitystreams", text/html;q=0.1'
 
+  attr_reader :response_code
+
   def call(url)
     return if url.blank?
 
@@ -27,6 +29,7 @@ class FetchResourceService < BaseService
   end
 
   def process_response(response, terminal = false)
+    @response_code = response.code
     return nil if response.code != 200
 
     if ['application/activity+json', 'application/ld+json'].include?(response.mime_type)
diff --git a/app/services/resolve_url_service.rb b/app/services/resolve_url_service.rb
index 1a2b0d60c..78080d878 100644
--- a/app/services/resolve_url_service.rb
+++ b/app/services/resolve_url_service.rb
@@ -12,7 +12,7 @@ class ResolveURLService < BaseService
       process_local_url
     elsif !fetched_resource.nil?
       process_url
-    elsif @on_behalf_of.present?
+    else
       process_url_from_db
     end
   end
@@ -30,6 +30,8 @@ class ResolveURLService < BaseService
   end
 
   def process_url_from_db
+    return unless @on_behalf_of.present? && [401, 403, 404].include?(fetch_resource_service.response_code)
+
     # It may happen that the resource is a private toot, and thus not fetchable,
     # but we can return the toot if we already know about it.
     status = Status.find_by(uri: @url) || Status.find_by(url: @url)
@@ -40,7 +42,11 @@ class ResolveURLService < BaseService
   end
 
   def fetched_resource
-    @fetched_resource ||= FetchResourceService.new.call(@url)
+    @fetched_resource ||= fetch_resource_service.call(@url)
+  end
+
+  def fetch_resource_service
+    @_fetch_resource_service ||= FetchResourceService.new
   end
 
   def resource_url
-- 
cgit 


From f08f880f584271a922a0d8d3759e634d67947d12 Mon Sep 17 00:00:00 2001
From: ThibG <thib@sitedethib.com>
Date: Wed, 25 Mar 2020 22:40:58 +0100
Subject: Fix media not being marked sensitive when client sets a CW but no
 text (#13277)
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

Mastodon enforces the “sensitive” flag on media attachments whenever a toot
is posted with a Content Warning. However, it does so *after* potentially
converting the Content Warning to toot text (when there is no toot text),
which leads to inconsistent and surprising behavior for API clients.
This commit fixes this inconsistency.
---
 app/services/post_status_service.rb       | 3 ++-
 spec/services/post_status_service_spec.rb | 7 +++++++
 2 files changed, 9 insertions(+), 1 deletion(-)

(limited to 'app/services')

diff --git a/app/services/post_status_service.rb b/app/services/post_status_service.rb
index c61b3baa2..0a383d6a3 100644
--- a/app/services/post_status_service.rb
+++ b/app/services/post_status_service.rb
@@ -48,6 +48,7 @@ class PostStatusService < BaseService
   private
 
   def preprocess_attributes!
+    @sensitive    = (@options[:sensitive].nil? ? @account.user&.setting_default_sensitive : @options[:sensitive]) || @options[:spoiler_text].present?
     @text         = @options.delete(:spoiler_text) if @text.blank? && @options[:spoiler_text].present?
     @visibility   = @options[:visibility] || @account.user&.setting_default_privacy
     @visibility   = :unlisted if @visibility&.to_sym == :public && @account.silenced?
@@ -157,7 +158,7 @@ class PostStatusService < BaseService
       media_attachments: @media || [],
       thread: @in_reply_to,
       poll_attributes: poll_attributes,
-      sensitive: (@options[:sensitive].nil? ? @account.user&.setting_default_sensitive : @options[:sensitive]) || @options[:spoiler_text].present?,
+      sensitive: @sensitive,
       spoiler_text: @options[:spoiler_text] || '',
       visibility: @visibility,
       language: language_from_option(@options[:language]) || @account.user&.setting_default_language&.presence || LanguageDetector.instance.detect(@text, @account),
diff --git a/spec/services/post_status_service_spec.rb b/spec/services/post_status_service_spec.rb
index 025a3da40..147a59fc3 100644
--- a/spec/services/post_status_service_spec.rb
+++ b/spec/services/post_status_service_spec.rb
@@ -79,6 +79,13 @@ RSpec.describe PostStatusService, type: :service do
     expect(status.spoiler_text).to eq spoiler_text
   end
 
+  it 'creates a sensitive status when there is a CW but no text' do
+    status = subject.call(Fabricate(:account), text: '', spoiler_text: 'foo')
+
+    expect(status).to be_persisted
+    expect(status).to be_sensitive
+  end
+
   it 'creates a status with empty default spoiler text' do
     status = create_status_with_options(spoiler_text: nil)
 
-- 
cgit 


From d98fabf2ee44c2c25775066abda46552ab05993e Mon Sep 17 00:00:00 2001
From: Thibaut Girka <thib@sitedethib.com>
Date: Fri, 27 Mar 2020 22:28:39 +0100
Subject: Fix crash when posting with a CW but no text nor media

---
 app/services/post_status_service.rb | 6 +++---
 1 file changed, 3 insertions(+), 3 deletions(-)

(limited to 'app/services')

diff --git a/app/services/post_status_service.rb b/app/services/post_status_service.rb
index 39b834604..250d0e8ed 100644
--- a/app/services/post_status_service.rb
+++ b/app/services/post_status_service.rb
@@ -50,11 +50,11 @@ class PostStatusService < BaseService
   def preprocess_attributes!
     if @text.blank? && @options[:spoiler_text].present?
      @text = '.'
-     if @media.find(&:video?) || @media.find(&:gifv?)
+     if @media&.find(&:video?) || @media&.find(&:gifv?)
        @text = '📹'
-     elsif @media.find(&:audio?)
+     elsif @media&.find(&:audio?)
        @text = '🎵'
-     elsif @media.find(&:image?)
+     elsif @media&.find(&:image?)
        @text = '🖼'
      end
     end
-- 
cgit 


From 11169367e4a999b19cc9140be38bdb5a1f3bb144 Mon Sep 17 00:00:00 2001
From: Takeshi Umeda <noel.yoshiba@gmail.com>
Date: Tue, 31 Mar 2020 03:32:34 +0900
Subject: Fix incorrect deletion of local accounts imported by overwriting
 (#13350)

---
 app/services/import_service.rb | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

(limited to 'app/services')

diff --git a/app/services/import_service.rb b/app/services/import_service.rb
index 4ee431ea3..c0d741d57 100644
--- a/app/services/import_service.rb
+++ b/app/services/import_service.rb
@@ -64,7 +64,8 @@ class ImportService < BaseService
   end
 
   def import_relationships!(action, undo_action, overwrite_scope, limit, extra_fields = {})
-    items = @data.take(limit).map { |row| [row['Account address']&.strip, Hash[extra_fields.map { |key, header| [key, row[header]&.strip] }]] }.reject { |(id, _)| id.blank? }
+    local_domain_suffix = "@#{Rails.configuration.x.local_domain}"
+    items = @data.take(limit).map { |row| [row['Account address']&.strip&.delete_suffix(local_domain_suffix), Hash[extra_fields.map { |key, header| [key, row[header]&.strip] }]] }.reject { |(id, _)| id.blank? }
 
     if @import.overwrite?
       presence_hash = items.each_with_object({}) { |(id, extra), mapping| mapping[id] = [true, extra] }
-- 
cgit 


From c3965e28b3ccc5ed31f37dec598bff646a8fb6d2 Mon Sep 17 00:00:00 2001
From: Eugen Rochko <eugen@zeonfederated.com>
Date: Thu, 2 Apr 2020 03:39:37 +0200
Subject: Fix returning results when searching for URL with non-zero offset
 (#13377)

Fix #13083
---
 app/services/search_service.rb | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

(limited to 'app/services')

diff --git a/app/services/search_service.rb b/app/services/search_service.rb
index 090fd409b..830de4de3 100644
--- a/app/services/search_service.rb
+++ b/app/services/search_service.rb
@@ -10,10 +10,10 @@ class SearchService < BaseService
     @resolve = options[:resolve] || false
 
     default_results.tap do |results|
-      next if @query.blank?
+      next if @query.blank? || @limit.zero?
 
       if url_query?
-        results.merge!(url_resource_results) unless url_resource.nil? || (@options[:type].present? && url_resource_symbol != @options[:type].to_sym)
+        results.merge!(url_resource_results) unless url_resource.nil? || @offset.positive? || (@options[:type].present? && url_resource_symbol != @options[:type].to_sym)
       elsif @query.present?
         results[:accounts] = perform_accounts_search! if account_searchable?
         results[:statuses] = perform_statuses_search! if full_text_searchable?
-- 
cgit