Merge branch 'master' into glitch-soc/merge-upstream

Conflicts:
- `README.md`:
  Our README.md files are completely different. Discarded upstream changes.
- `app/javascript/core/admin.js`:
  Updating rails-ujs, no real conflict, but a comment to close to changed
  code. Various glitch-soc-only files have been updated to match those changes,
  though.
- `package.json`:
  No real conflict, just an additional dependency in glitch-soc that was too
  close to something updated upstream. Took upstream's changes.

8 files changed, 113 insertions, 47 deletions

diff --git a/app/models/account.rb b/app/models/account.rb
index 73421ee5a..82d4d10de 100644
--- a/app/models/account.rb
+++ b/app/models/account.rb
@@ -46,6 +46,7 @@
 #  silenced_at             :datetime
 #  suspended_at            :datetime
 #  trust_level             :integer
+#  hide_collections        :boolean
 #
 
 class Account < ApplicationRecord
@@ -325,6 +326,14 @@ class Account < ApplicationRecord
     save!
   end
 
+  def hides_followers?
+    hide_collections? || user_hides_network?
+  end
+
+  def hides_following?
+    hide_collections? || user_hides_network?
+  end
+
   def object_type
     :person
   end
diff --git a/app/models/account_warning_preset.rb b/app/models/account_warning_preset.rb
index ba8ceabb3..c20f683cf 100644
--- a/app/models/account_warning_preset.rb
+++ b/app/models/account_warning_preset.rb
@@ -8,8 +8,11 @@
 #  text       :text             default(""), not null
 #  created_at :datetime         not null
 #  updated_at :datetime         not null
+#  title      :string           default(""), not null
 #
 
 class AccountWarningPreset < ApplicationRecord
   validates :text, presence: true
+
+  scope :alphabetic, -> { order(title: :asc, text: :asc) }
 end
diff --git a/app/models/admin/account_action.rb b/app/models/admin/account_action.rb
index e9da003a3..b30a82369 100644
--- a/app/models/admin/account_action.rb
+++ b/app/models/admin/account_action.rb
@@ -62,8 +62,6 @@ class Admin::AccountAction
 
   def process_action!
     case type
-    when 'none'
-      handle_resolve!
     when 'disable'
       handle_disable!
     when 'silence'
@@ -105,16 +103,6 @@ class Admin::AccountAction
     end
   end
 
-  def handle_resolve!
-    if with_report? && report.account_id == -99 && target_account.trust_level == Account::TRUST_LEVELS[:untrusted]
-      # This is an automated report and it is being dismissed, so it's
-      # a false positive, in which case update the account's trust level
-      # to prevent further spam checks
-
-      target_account.update(trust_level: Account::TRUST_LEVELS[:trusted])
-    end
-  end
-
   def handle_disable!
     authorize(target_account.user, :disable?)
     log_action(:disable, target_account.user)
diff --git a/app/models/concerns/attachmentable.rb b/app/models/concerns/attachmentable.rb
index 43ff8ac12..18b872c1e 100644
--- a/app/models/concerns/attachmentable.rb
+++ b/app/models/concerns/attachmentable.rb
@@ -74,7 +74,7 @@ module Attachmentable
     self.class.attachment_definitions.each_key do |attachment_name|
       attachment = send(attachment_name)
 
-      next if attachment.blank? || attachment.queued_for_write[:original].blank?
+      next if attachment.blank? || attachment.queued_for_write[:original].blank? || attachment.options[:preserve_files]
 
       attachment.instance_write :file_name, SecureRandom.hex(8) + File.extname(attachment.instance_read(:file_name))
     end
diff --git a/app/models/email_domain_block.rb b/app/models/email_domain_block.rb
index bc70dea25..f50fa46ba 100644
--- a/app/models/email_domain_block.rb
+++ b/app/models/email_domain_block.rb
@@ -7,13 +7,27 @@
 #  domain     :string           default(""), not null
 #  created_at :datetime         not null
 #  updated_at :datetime         not null
+#  parent_id  :bigint(8)
 #
 
 class EmailDomainBlock < ApplicationRecord
   include DomainNormalizable
 
+  belongs_to :parent, class_name: 'EmailDomainBlock', optional: true
+  has_many :children, class_name: 'EmailDomainBlock', foreign_key: :parent_id, inverse_of: :parent, dependent: :destroy
+
   validates :domain, presence: true, uniqueness: true, domain: true
 
+  def with_dns_records=(val)
+    @with_dns_records = ActiveModel::Type::Boolean.new.cast(val)
+  end
+
+  def with_dns_records?
+    @with_dns_records
+  end
+
+  alias with_dns_records with_dns_records?
+
   def self.block?(email)
     _, domain = email.split('@', 2)
 
diff --git a/app/models/media_attachment.rb b/app/models/media_attachment.rb
index 2813d9200..59427a8cc 100644
--- a/app/models/media_attachment.rb
+++ b/app/models/media_attachment.rb
@@ -19,12 +19,14 @@
 #  description         :text
 #  scheduled_status_id :bigint(8)
 #  blurhash            :string
+#  processing          :integer
 #
 
 class MediaAttachment < ApplicationRecord
   self.inheritance_column = nil
 
   enum type: [:image, :gifv, :video, :unknown, :audio]
+  enum processing: [:queued, :in_progress, :complete, :failed], _prefix: true
 
   MAX_DESCRIPTION_LENGTH = 1_500
 
@@ -55,6 +57,43 @@ class MediaAttachment < ApplicationRecord
     },
   }.freeze
 
+  VIDEO_FORMAT = {
+    format: 'mp4',
+    content_type: 'video/mp4',
+    convert_options: {
+      output: {
+        'loglevel' => 'fatal',
+        'movflags' => 'faststart',
+        'pix_fmt' => 'yuv420p',
+        'vf' => 'scale=\'trunc(iw/2)*2:trunc(ih/2)*2\'',
+        'vsync' => 'cfr',
+        'c:v' => 'h264',
+        'maxrate' => '1300K',
+        'bufsize' => '1300K',
+        'frames:v' => 60 * 60 * 3,
+        'crf' => 18,
+        'map_metadata' => '-1',
+      },
+    },
+  }.freeze
+
+  VIDEO_PASSTHROUGH_OPTIONS = {
+    video_codecs: ['h264'],
+    audio_codecs: ['aac', nil],
+    colorspaces: ['yuv420p'],
+    options: {
+      format: 'mp4',
+      convert_options: {
+        output: {
+          'loglevel' => 'fatal',
+          'map_metadata' => '-1',
+          'c:v' => 'copy',
+          'c:a' => 'copy',
+        },
+      },
+    },
+  }.freeze
+
   VIDEO_STYLES = {
     small: {
       convert_options: {
@@ -69,17 +108,7 @@ class MediaAttachment < ApplicationRecord
       blurhash: BLURHASH_OPTIONS,
     },
 
-    original: {
-      keep_same_format: true,
-      convert_options: {
-        output: {
-          'loglevel' => 'fatal',
-          'map_metadata' => '-1',
-          'c:v' => 'copy',
-          'c:a' => 'copy',
-        },
-      },
-    },
+    original: VIDEO_FORMAT.merge(passthrough_options: VIDEO_PASSTHROUGH_OPTIONS),
   }.freeze
 
   AUDIO_STYLES = {
@@ -96,26 +125,6 @@ class MediaAttachment < ApplicationRecord
     },
   }.freeze
 
-  VIDEO_FORMAT = {
-    format: 'mp4',
-    content_type: 'video/mp4',
-    convert_options: {
-      output: {
-        'loglevel' => 'fatal',
-        'movflags' => 'faststart',
-        'pix_fmt' => 'yuv420p',
-        'vf' => 'scale=\'trunc(iw/2)*2:trunc(ih/2)*2\'',
-        'vsync' => 'cfr',
-        'c:v' => 'h264',
-        'maxrate' => '1300K',
-        'bufsize' => '1300K',
-        'frames:v' => 60 * 60 * 3,
-        'crf' => 18,
-        'map_metadata' => '-1',
-      },
-    },
-  }.freeze
-
   VIDEO_CONVERTED_STYLES = {
     small: VIDEO_STYLES[:small],
     original: VIDEO_FORMAT,
@@ -124,6 +133,9 @@ class MediaAttachment < ApplicationRecord
   IMAGE_LIMIT = (ENV['MAX_IMAGE_SIZE'] || 10.megabytes).to_i
   VIDEO_LIMIT = (ENV['MAX_VIDEO_SIZE'] || 40.megabytes).to_i
 
+  MAX_VIDEO_MATRIX_LIMIT = 2_304_000 # 1920x1200px
+  MAX_VIDEO_FRAME_RATE   = 60
+
   belongs_to :account,          inverse_of: :media_attachments, optional: true
   belongs_to :status,           inverse_of: :media_attachments, optional: true
   belongs_to :scheduled_status, inverse_of: :media_attachments, optional: true
@@ -156,6 +168,10 @@ class MediaAttachment < ApplicationRecord
     remote_url.blank?
   end
 
+  def not_processed?
+    processing.present? && !processing_complete?
+  end
+
   def needs_redownload?
     file.blank? && remote_url.present?
   end
@@ -203,12 +219,21 @@ class MediaAttachment < ApplicationRecord
     "#{x},#{y}"
   end
 
+  attr_writer :delay_processing
+
+  def delay_processing?
+    @delay_processing
+  end
+
+  after_commit :enqueue_processing, on: :create
   after_commit :reset_parent_cache, on: :update
 
   before_create :prepare_description, unless: :local?
   before_create :set_shortcode
+  before_create :set_processing
 
   before_post_process :set_type_and_extension
+  before_post_process :check_video_dimensions
 
   before_save :set_meta
 
@@ -277,6 +302,21 @@ class MediaAttachment < ApplicationRecord
     end
   end
 
+  def set_processing
+    self.processing = delay_processing? ? :queued : :complete
+  end
+
+  def check_video_dimensions
+    return unless (video? || gifv?) && file.queued_for_write[:original].present?
+
+    movie = FFMPEG::Movie.new(file.queued_for_write[:original].path)
+
+    return unless movie.valid?
+
+    raise Mastodon::DimensionsValidationError, "#{movie.width}x#{movie.height} videos are not supported" if movie.width * movie.height > MAX_VIDEO_MATRIX_LIMIT
+    raise Mastodon::DimensionsValidationError, "#{movie.frame_rate.to_i}fps videos are not supported" if movie.frame_rate > MAX_VIDEO_FRAME_RATE
+  end
+
   def set_meta
     meta = populate_meta
 
@@ -322,9 +362,11 @@ class MediaAttachment < ApplicationRecord
     }.compact
   end
 
-  def reset_parent_cache
-    return if status_id.nil?
+  def enqueue_processing
+    PostProcessMediaWorker.perform_async(id) if delay_processing?
+  end
 
-    Rails.cache.delete("statuses/#{status_id}")
+  def reset_parent_cache
+    Rails.cache.delete("statuses/#{status_id}") if status_id.present?
   end
 end
diff --git a/app/models/report.rb b/app/models/report.rb
index fb2e040ee..356c23d68 100644
--- a/app/models/report.rb
+++ b/app/models/report.rb
@@ -59,6 +59,14 @@ class Report < ApplicationRecord
   end
 
   def resolve!(acting_account)
+    if account_id == -99 && target_account.trust_level == Account::TRUST_LEVELS[:untrusted]
+      # This is an automated report and it is being dismissed, so it's
+      # a false positive, in which case update the account's trust level
+      # to prevent further spam checks
+
+      target_account.update(trust_level: Account::TRUST_LEVELS[:trusted])
+    end
+
     RemovalWorker.push_bulk(Status.with_discarded.discarded.where(id: status_ids).pluck(:id)) { |status_id| [status_id, { immediate: true }] }
     update!(action_taken: true, action_taken_by_account_id: acting_account.id)
   end
diff --git a/app/models/status.rb b/app/models/status.rb
index b2d3c3f3b..a78717d0c 100644
--- a/app/models/status.rb
+++ b/app/models/status.rb
@@ -148,10 +148,12 @@ class Status < ApplicationRecord
       ids += mentions.where(account: Account.local).pluck(:account_id)
       ids += favourites.where(account: Account.local).pluck(:account_id)
       ids += reblogs.where(account: Account.local).pluck(:account_id)
+      ids += bookmarks.where(account: Account.local).pluck(:account_id)
     else
       ids += preloaded.mentions[id] || []
       ids += preloaded.favourites[id] || []
       ids += preloaded.reblogs[id] || []
+      ids += preloaded.bookmarks[id] || []
     end
 
     ids.uniq