Merge branch 'master' into glitch-soc/merge-upstream

Conflicts:
- app/models/media_attachment.rb
  Upstream added audio attachment support
- app/serializers/initial_state_serializer.rb
  Upstream added audio attachment support and how mimetypes are returned
- app/serializers/rest/instance_serializer.rb
  Upstream added a few fields
- config/application.rb
  Upstream added a different paperclip transcoder

11 files changed, 136 insertions, 56 deletions

diff --git a/app/models/account.rb b/app/models/account.rb
index 520b183e8..3d7b0dda3 100644
--- a/app/models/account.rb
+++ b/app/models/account.rb
@@ -102,6 +102,7 @@ class Account < ApplicationRecord
   scope :tagged_with, ->(tag) { joins(:accounts_tags).where(accounts_tags: { tag_id: tag }) }
   scope :by_recent_status, -> { order(Arel.sql('(case when account_stats.last_status_at is null then 1 else 0 end) asc, account_stats.last_status_at desc')) }
   scope :popular, -> { order('account_stats.followers_count desc') }
+  scope :by_domain_and_subdomains, ->(domain) { where(domain: domain).or(where(arel_table[:domain].matches('%.' + domain))) }
 
   delegate :email,
            :unconfirmed_email,
@@ -110,6 +111,8 @@ class Account < ApplicationRecord
            :confirmed?,
            :approved?,
            :pending?,
+           :disabled?,
+           :role,
            :admin?,
            :moderator?,
            :staff?,
diff --git a/app/models/account_filter.rb b/app/models/account_filter.rb
index d2503100c..c3b1fe08d 100644
--- a/app/models/account_filter.rb
+++ b/app/models/account_filter.rb
@@ -37,6 +37,8 @@ class AccountFilter
       Account.without_suspended
     when 'pending'
       accounts_with_users.merge User.pending
+    when 'disabled'
+      accounts_with_users.merge User.disabled
     when 'silenced'
       Account.silenced
     when 'suspended'
diff --git a/app/models/concerns/attachmentable.rb b/app/models/concerns/attachmentable.rb
index de4cf8775..24f5968de 100644
--- a/app/models/concerns/attachmentable.rb
+++ b/app/models/concerns/attachmentable.rb
@@ -1,6 +1,6 @@
 # frozen_string_literal: true
 
-require 'mime/types'
+require 'mime/types/columnar'
 
 module Attachmentable
   extend ActiveSupport::Concern
@@ -10,10 +10,21 @@ module Attachmentable
   included do
     before_post_process :set_file_extensions
     before_post_process :check_image_dimensions
+    before_post_process :set_file_content_type
   end
 
   private
 
+  def set_file_content_type
+    self.class.attachment_definitions.each_key do |attachment_name|
+      attachment = send(attachment_name)
+
+      next if attachment.blank? || attachment.queued_for_write[:original].blank?
+
+      attachment.instance_write :content_type, calculated_content_type(attachment)
+    end
+  end
+
   def set_file_extensions
     self.class.attachment_definitions.each_key do |attachment_name|
       attachment = send(attachment_name)
@@ -47,4 +58,10 @@ module Attachmentable
 
     extension
   end
+
+  def calculated_content_type(attachment)
+    Paperclip.run('file', '-b --mime :file', file: attachment.queued_for_write[:original].path).split(/[:;\s]+/).first.chomp
+  rescue Terrapin::CommandLineError
+    ''
+  end
 end
diff --git a/app/models/concerns/user_roles.rb b/app/models/concerns/user_roles.rb
index 58dffdc46..a42b4a172 100644
--- a/app/models/concerns/user_roles.rb
+++ b/app/models/concerns/user_roles.rb
@@ -13,6 +13,20 @@ module UserRoles
     admin? || moderator?
   end
 
+  def role=(value)
+    case value
+    when 'admin'
+      self.admin     = true
+      self.moderator = false
+    when 'moderator'
+      self.admin     = false
+      self.moderator = true
+    else
+      self.admin     = false
+      self.moderator = false
+    end
+  end
+
   def role
     if admin?
       'admin'
diff --git a/app/models/custom_emoji.rb b/app/models/custom_emoji.rb
index d3cc70504..e73cd9bd2 100644
--- a/app/models/custom_emoji.rb
+++ b/app/models/custom_emoji.rb
@@ -39,6 +39,7 @@ class CustomEmoji < ApplicationRecord
   scope :local,      -> { where(domain: nil) }
   scope :remote,     -> { where.not(domain: nil) }
   scope :alphabetic, -> { order(domain: :asc, shortcode: :asc) }
+  scope :by_domain_and_subdomains, ->(domain) { where(domain: domain).or(where(arel_table[:domain].matches('%.' + domain))) }
 
   remotable_attachment :image, LIMIT
 
diff --git a/app/models/domain_block.rb b/app/models/domain_block.rb
index 84c08c158..25d3b87ef 100644
--- a/app/models/domain_block.rb
+++ b/app/models/domain_block.rb
@@ -24,14 +24,41 @@ class DomainBlock < ApplicationRecord
 
   scope :matches_domain, ->(value) { where(arel_table[:domain].matches("%#{value}%")) }
 
-  def self.blocked?(domain)
-    where(domain: domain, severity: :suspend).exists?
+  class << self
+    def suspend?(domain)
+      !!rule_for(domain)&.suspend?
+    end
+
+    def silence?(domain)
+      !!rule_for(domain)&.silence?
+    end
+
+    def reject_media?(domain)
+      !!rule_for(domain)&.reject_media?
+    end
+
+    def reject_reports?(domain)
+      !!rule_for(domain)&.reject_reports?
+    end
+
+    alias blocked? suspend?
+
+    def rule_for(domain)
+      return if domain.blank?
+
+      uri      = Addressable::URI.new.tap { |u| u.host = domain.gsub(/[\/]/, '') }
+      segments = uri.normalized_host.split('.')
+      variants = segments.map.with_index { |_, i| segments[i..-1].join('.') }
+
+      where(domain: variants[0..-2]).order(Arel.sql('char_length(domain) desc')).first
+    end
   end
 
   def stricter_than?(other_block)
-    return true if suspend?
+    return true  if suspend?
     return false if other_block.suspend? && (silence? || noop?)
     return false if other_block.silence? && noop?
+
     (reject_media || !other_block.reject_media) && (reject_reports || !other_block.reject_reports)
   end
 
diff --git a/app/models/instance.rb b/app/models/instance.rb
index 7bf000d40..797a191e0 100644
--- a/app/models/instance.rb
+++ b/app/models/instance.rb
@@ -8,15 +8,11 @@ class Instance
   def initialize(resource)
     @domain         = resource.domain
     @accounts_count = resource.is_a?(DomainBlock) ? nil : resource.accounts_count
-    @domain_block   = resource.is_a?(DomainBlock) ? resource : DomainBlock.find_by(domain: domain)
+    @domain_block   = resource.is_a?(DomainBlock) ? resource : DomainBlock.rule_for(domain)
   end
 
-  def cached_sample_accounts
-    Rails.cache.fetch("#{cache_key}/sample_accounts", expires_in: 12.hours) { Account.where(domain: domain).searchable.joins(:account_stat).popular.limit(3) }
-  end
-
-  def cached_accounts_count
-    @accounts_count || Rails.cache.fetch("#{cache_key}/count", expires_in: 12.hours) { Account.where(domain: domain).count }
+  def countable?
+    @accounts_count.present?
   end
 
   def to_param
diff --git a/app/models/media_attachment.rb b/app/models/media_attachment.rb
index 70a671b4a..815ac0258 100644
--- a/app/models/media_attachment.rb
+++ b/app/models/media_attachment.rb
@@ -24,16 +24,16 @@
 class MediaAttachment < ApplicationRecord
   self.inheritance_column = nil
 
-  enum type: [:image, :gifv, :video, :audio, :unknown]
+  enum type: [:image, :gifv, :video, :unknown, :audio]
 
   IMAGE_FILE_EXTENSIONS = ['.jpg', '.jpeg', '.png', '.gif', '.webp'].freeze
   VIDEO_FILE_EXTENSIONS = ['.webm', '.mp4', '.m4v', '.mov'].freeze
-  AUDIO_FILE_EXTENSIONS = ['.mp3', '.m4a', '.wav', '.ogg'].freeze
+  AUDIO_FILE_EXTENSIONS = ['.ogg', '.oga', '.mp3', '.m4a', '.wav', '.flac', '.opus'].freeze
 
   IMAGE_MIME_TYPES             = ['image/jpeg', 'image/png', 'image/gif', 'image/webp'].freeze
-  VIDEO_MIME_TYPES             = ['video/webm', 'video/mp4', 'video/quicktime'].freeze
+  VIDEO_MIME_TYPES             = ['video/webm', 'video/mp4', 'video/quicktime', 'video/ogg'].freeze
   VIDEO_CONVERTIBLE_MIME_TYPES = ['video/webm', 'video/quicktime'].freeze
-  AUDIO_MIME_TYPES             = ['audio/mpeg', 'audio/mp4', 'audio/vnd.wav', 'audio/wav', 'audio/x-wav', 'audio/x-wave', 'audio/ogg',].freeze
+  AUDIO_MIME_TYPES             = ['audio/wave', 'audio/wav', 'audio/x-wav', 'audio/x-wave', 'audio/vdn.wav', 'audio/x-pn-wave', 'audio/ogg', 'audio/mpeg', 'audio/mp3', 'audio/mp4', 'audio/webm', 'audio/flac'].freeze
 
   BLURHASH_OPTIONS = {
     x_comp: 4,
@@ -53,22 +53,6 @@ class MediaAttachment < ApplicationRecord
     },
   }.freeze
 
-  AUDIO_STYLES = {
-    original: {
-      format: 'mp4',
-      convert_options: {
-        output: {
-          filter_complex: '"[0:a]compand,showwaves=s=640x360:mode=line,format=yuv420p[v]"',
-          map: '"[v]" -map 0:a', 
-          threads: 2,
-          vcodec: 'libx264',
-          acodec: 'aac',
-          movflags: '+faststart',
-        },
-      },
-    },
-  }.freeze
-
   VIDEO_STYLES = {
     small: {
       convert_options: {
@@ -83,8 +67,21 @@ class MediaAttachment < ApplicationRecord
     },
   }.freeze
 
+  AUDIO_STYLES = {
+    original: {
+      format: 'mp3',
+      content_type: 'audio/mpeg',
+      convert_options: {
+        output: {
+          'q:a' => 2,
+        },
+      },
+    },
+  }.freeze
+
   VIDEO_FORMAT = {
     format: 'mp4',
+    content_type: 'video/mp4',
     convert_options: {
       output: {
         'loglevel' => 'fatal',
@@ -101,6 +98,11 @@ class MediaAttachment < ApplicationRecord
     },
   }.freeze
 
+  VIDEO_CONVERTED_STYLES = {
+    small: VIDEO_STYLES[:small],
+    original: VIDEO_FORMAT,
+  }.freeze
+
   IMAGE_LIMIT = (ENV['MAX_IMAGE_SIZE'] || 8.megabytes).to_i
   VIDEO_LIMIT = (ENV['MAX_VIDEO_SIZE'] || 40.megabytes).to_i
 
@@ -114,8 +116,8 @@ class MediaAttachment < ApplicationRecord
                     convert_options: { all: '-quality 90 -strip' }
 
   validates_attachment_content_type :file, content_type: IMAGE_MIME_TYPES + VIDEO_MIME_TYPES + AUDIO_MIME_TYPES
-  validates_attachment_size :file, less_than: IMAGE_LIMIT, unless: :video_or_gifv?
-  validates_attachment_size :file, less_than: VIDEO_LIMIT, if: :video_or_gifv?
+  validates_attachment_size :file, less_than: IMAGE_LIMIT, unless: :larger_media_format?
+  validates_attachment_size :file, less_than: VIDEO_LIMIT, if: :larger_media_format?
   remotable_attachment :file, VIDEO_LIMIT
 
   include Attachmentable
@@ -138,8 +140,12 @@ class MediaAttachment < ApplicationRecord
     file.blank? && remote_url.present?
   end
 
-  def video_or_gifv?
-    video? || gifv?
+  def larger_media_format?
+    video? || gifv? || audio?
+  end
+
+  def audio_or_video?
+    audio? || video?
   end
 
   def to_param
@@ -171,37 +177,37 @@ class MediaAttachment < ApplicationRecord
   before_save :set_meta
 
   class << self
+    def supported_mime_types
+      IMAGE_MIME_TYPES + VIDEO_MIME_TYPES + AUDIO_MIME_TYPES
+    end
+
+    def supported_file_extensions
+      IMAGE_FILE_EXTENSIONS + VIDEO_FILE_EXTENSIONS + AUDIO_FILE_EXTENSIONS
+    end
+
     private
 
     def file_styles(f)
-      if f.instance.file_content_type == 'image/gif'
-        {
-          small: IMAGE_STYLES[:small],
-          original: VIDEO_FORMAT,
-        }
-      elsif IMAGE_MIME_TYPES.include? f.instance.file_content_type
+      if f.instance.file_content_type == 'image/gif' || VIDEO_CONVERTIBLE_MIME_TYPES.include?(f.instance.file_content_type)
+        VIDEO_CONVERTED_STYLES
+      elsif IMAGE_MIME_TYPES.include?(f.instance.file_content_type)
         IMAGE_STYLES
-      elsif AUDIO_MIME_TYPES.include? f.instance.file_content_type
-        AUDIO_STYLES
-      elsif VIDEO_CONVERTIBLE_MIME_TYPES.include?(f.instance.file_content_type)
-        {
-          small: VIDEO_STYLES[:small],
-          original: VIDEO_FORMAT,
-        }
-      else
+      elsif VIDEO_MIME_TYPES.include?(f.instance.file_content_type)
         VIDEO_STYLES
+      else
+        AUDIO_STYLES
       end
     end
 
     def file_processors(f)
       if f.file_content_type == 'image/gif'
         [:gif_transcoder, :blurhash_transcoder]
-      elsif VIDEO_MIME_TYPES.include? f.file_content_type
-        [:video_transcoder, :blurhash_transcoder]
-      elsif AUDIO_MIME_TYPES.include? f.file_content_type
-        [:audio_transcoder]
+      elsif VIDEO_MIME_TYPES.include?(f.file_content_type)
+        [:video_transcoder, :blurhash_transcoder, :type_corrector]
+      elsif AUDIO_MIME_TYPES.include?(f.file_content_type)
+        [:transcoder, :type_corrector]
       else
-        [:lazy_thumbnail, :blurhash_transcoder]
+        [:lazy_thumbnail, :blurhash_transcoder, :type_corrector]
       end
     end
   end
@@ -224,7 +230,15 @@ class MediaAttachment < ApplicationRecord
   end
 
   def set_type_and_extension
-    self.type = VIDEO_MIME_TYPES.include?(file_content_type) ? :video : AUDIO_MIME_TYPES.include?(file_content_type) ? :audio : :image
+    self.type = begin
+      if VIDEO_MIME_TYPES.include?(file_content_type)
+        :video
+      elsif AUDIO_MIME_TYPES.include?(file_content_type)
+        :audio
+      else
+        :image
+      end
+    end
   end
 
   def set_meta
@@ -267,7 +281,7 @@ class MediaAttachment < ApplicationRecord
       frame_rate: movie.frame_rate,
       duration: movie.duration,
       bitrate: movie.bitrate,
-    }
+    }.compact
   end
 
   def reset_parent_cache
diff --git a/app/models/report.rb b/app/models/report.rb
index 86c303798..5192ceef7 100644
--- a/app/models/report.rb
+++ b/app/models/report.rb
@@ -17,6 +17,8 @@
 #
 
 class Report < ApplicationRecord
+  include Paginable
+
   belongs_to :account
   belongs_to :target_account, class_name: 'Account'
   belongs_to :action_taken_by_account, class_name: 'Account', optional: true
@@ -26,6 +28,7 @@ class Report < ApplicationRecord
 
   scope :unresolved, -> { where(action_taken: false) }
   scope :resolved,   -> { where(action_taken: true) }
+  scope :with_accounts, -> { includes([:account, :target_account, :action_taken_by_account, :assigned_account].each_with_object({}) { |k, h| h[k] = { user: [:invite_request, :invite] } }) }
 
   validates :comment, length: { maximum: 1000 }
 
diff --git a/app/models/report_filter.rb b/app/models/report_filter.rb
index 56ab28df7..a392d60c3 100644
--- a/app/models/report_filter.rb
+++ b/app/models/report_filter.rb
@@ -9,9 +9,11 @@ class ReportFilter
 
   def results
     scope = Report.unresolved
+
     params.each do |key, value|
       scope = scope.merge scope_for(key, value)
     end
+
     scope
   end
 
diff --git a/app/models/user.rb b/app/models/user.rb
index fcfb79612..f6936cb9d 100644
--- a/app/models/user.rb
+++ b/app/models/user.rb
@@ -87,6 +87,7 @@ class User < ApplicationRecord
   scope :approved, -> { where(approved: true) }
   scope :confirmed, -> { where.not(confirmed_at: nil) }
   scope :enabled, -> { where(disabled: false) }
+  scope :disabled, -> { where(disabled: true) }
   scope :inactive, -> { where(arel_table[:current_sign_in_at].lt(ACTIVE_DURATION.ago)) }
   scope :active, -> { confirmed.where(arel_table[:current_sign_in_at].gteq(ACTIVE_DURATION.ago)).joins(:account).where(accounts: { suspended_at: nil }) }
   scope :matches_email, ->(value) { where(arel_table[:email].matches("#{value}%")) }