Merge branch 'master' into glitch-soc/merge-upstream

Conflicts:
- Gemfile
- app/controllers/api/v1/search_controller.rb
  Conflict because we changed the number of default results to be
  configurable
- app/lib/settings/scoped_settings.rb
  Addition of a new “noindex” site-wide setting,
  conflict due to our change of the two other site-wide settings
  (default flavour and skin instead of theme)
- spec/controllers/application_controller_spec.rb
  Addition of a new “noindex” site-wide setting,
  conflict due to our change of the two other site-wide settings
  (default flavour and skin instead of theme)

19 files changed, 229 insertions, 24 deletions

diff --git a/app/models/account.rb b/app/models/account.rb
index 918b17430..e1593fe81 100644
--- a/app/models/account.rb
+++ b/app/models/account.rb
@@ -119,6 +119,7 @@ class Account < ApplicationRecord
            :approved?,
            :pending?,
            :disabled?,
+           :unconfirmed_or_pending?,
            :role,
            :admin?,
            :moderator?,
diff --git a/app/models/admin/account_action.rb b/app/models/admin/account_action.rb
index c7da8b52c..b30a82369 100644
--- a/app/models/admin/account_action.rb
+++ b/app/models/admin/account_action.rb
@@ -83,19 +83,23 @@ class Admin::AccountAction
 
     # A log entry is only interesting if the warning contains
     # custom text from someone. Otherwise it's just noise.
+
     log_action(:create, warning) if warning.text.present?
   end
 
   def process_reports!
-    return if report_id.blank?
+    # If we're doing "mark as resolved" on a single report,
+    # then we want to keep other reports open in case they
+    # contain new actionable information.
+    #
+    # Otherwise, we will mark all unresolved reports about
+    # the account as resolved.
 
-    authorize(report, :update?)
+    reports.each { |report| authorize(report, :update?) }
 
-    if type == 'none'
+    reports.each do |report|
       log_action(:resolve, report)
       report.resolve!(current_account)
-    else
-      Report.where(target_account: target_account).unresolved.update_all(action_taken: true, action_taken_by_account_id: current_account.id)
     end
   end
 
@@ -141,6 +145,16 @@ class Admin::AccountAction
     @report.status_ids if @report && include_statuses
   end
 
+  def reports
+    @reports ||= begin
+      if type == 'none' && with_report?
+        [report]
+      else
+        Report.where(target_account: target_account).unresolved
+      end
+    end
+  end
+
   def warning_preset
     @warning_preset ||= AccountWarningPreset.find(warning_preset_id) if warning_preset_id.present?
   end
diff --git a/app/models/concerns/remotable.rb b/app/models/concerns/remotable.rb
index 9372a963b..082302619 100644
--- a/app/models/concerns/remotable.rb
+++ b/app/models/concerns/remotable.rb
@@ -4,7 +4,7 @@ module Remotable
   extend ActiveSupport::Concern
 
   class_methods do
-    def remotable_attachment(attachment_name, limit)
+    def remotable_attachment(attachment_name, limit, suppress_errors: true)
       attribute_name  = "#{attachment_name}_remote_url".to_sym
       method_name     = "#{attribute_name}=".to_sym
       alt_method_name = "reset_#{attachment_name}!".to_sym
@@ -22,7 +22,7 @@ module Remotable
 
         begin
           Request.new(:get, url).perform do |response|
-            next if response.code != 200
+            raise Mastodon::UnexpectedResponseError, response unless (200...300).cover?(response.code)
 
             content_type = parse_content_type(response.headers.get('content-type').last)
             extname      = detect_extname_from_content_type(content_type)
@@ -41,11 +41,11 @@ module Remotable
 
             self[attribute_name] = url if has_attribute?(attribute_name)
           end
-        rescue HTTP::TimeoutError, HTTP::ConnectionError, OpenSSL::SSL::SSLError, Paperclip::Errors::NotIdentifiedByImageMagickError, Addressable::URI::InvalidURIError, Mastodon::HostValidationError, Mastodon::LengthValidationError => e
+        rescue Mastodon::UnexpectedResponseError, HTTP::TimeoutError, HTTP::ConnectionError, OpenSSL::SSL::SSLError => e
+          Rails.logger.debug "Error fetching remote #{attachment_name}: #{e}"
+          raise e unless suppress_errors
+        rescue Paperclip::Errors::NotIdentifiedByImageMagickError, Addressable::URI::InvalidURIError, Mastodon::HostValidationError, Mastodon::LengthValidationError, Paperclip::Error, Mastodon::DimensionsValidationError => e
           Rails.logger.debug "Error fetching remote #{attachment_name}: #{e}"
-          nil
-        rescue Paperclip::Error, Mastodon::DimensionsValidationError => e
-          Rails.logger.debug "Error processing remote #{attachment_name}: #{e}"
           nil
         end
       end
diff --git a/app/models/custom_emoji.rb b/app/models/custom_emoji.rb
index b21ad9042..edb1bec75 100644
--- a/app/models/custom_emoji.rb
+++ b/app/models/custom_emoji.rb
@@ -40,10 +40,11 @@ class CustomEmoji < ApplicationRecord
   validates_attachment :image, content_type: { content_type: IMAGE_MIME_TYPES }, presence: true, size: { less_than: LIMIT }
   validates :shortcode, uniqueness: { scope: :domain }, format: { with: /\A#{SHORTCODE_RE_FRAGMENT}\z/ }, length: { minimum: 2 }
 
-  scope :local,      -> { where(domain: nil) }
-  scope :remote,     -> { where.not(domain: nil) }
+  scope :local, -> { where(domain: nil) }
+  scope :remote, -> { where.not(domain: nil) }
   scope :alphabetic, -> { order(domain: :asc, shortcode: :asc) }
   scope :by_domain_and_subdomains, ->(domain) { where(domain: domain).or(where(arel_table[:domain].matches('%.' + domain))) }
+  scope :listed, -> { local.where(disabled: false).where(visible_in_picker: true) }
 
   remotable_attachment :image, LIMIT
 
@@ -59,6 +60,12 @@ class CustomEmoji < ApplicationRecord
     :emoji
   end
 
+  def copy!
+    copy = self.class.find_or_initialize_by(domain: nil, shortcode: shortcode)
+    copy.image = image
+    copy.save!
+  end
+
   class << self
     def from_text(text, domain)
       return [] if text.blank?
diff --git a/app/models/custom_emoji_category.rb b/app/models/custom_emoji_category.rb
index 7d8c0ee2d..3c87f2b2e 100644
--- a/app/models/custom_emoji_category.rb
+++ b/app/models/custom_emoji_category.rb
@@ -12,4 +12,6 @@
 
 class CustomEmojiCategory < ApplicationRecord
   has_many :emojis, class_name: 'CustomEmoji', foreign_key: 'category_id', inverse_of: :category
+
+  validates :name, presence: true, uniqueness: true
 end
diff --git a/app/models/custom_emoji_filter.rb b/app/models/custom_emoji_filter.rb
index 7649055d2..15b8da1d1 100644
--- a/app/models/custom_emoji_filter.rb
+++ b/app/models/custom_emoji_filter.rb
@@ -11,6 +11,8 @@ class CustomEmojiFilter
     scope = CustomEmoji.alphabetic
 
     params.each do |key, value|
+      next if key.to_s == 'page'
+
       scope.merge!(scope_for(key, value)) if value.present?
     end
 
@@ -22,13 +24,13 @@ class CustomEmojiFilter
   def scope_for(key, value)
     case key.to_s
     when 'local'
-      CustomEmoji.local
+      CustomEmoji.local.left_joins(:category).reorder(Arel.sql('custom_emoji_categories.name ASC NULLS FIRST, custom_emojis.shortcode ASC'))
     when 'remote'
       CustomEmoji.remote
     when 'by_domain'
-      CustomEmoji.where(domain: value.downcase)
+      CustomEmoji.where(domain: value.strip.downcase)
     when 'shortcode'
-      CustomEmoji.search(value)
+      CustomEmoji.search(value.strip)
     else
       raise "Unknown filter: #{key}"
     end
diff --git a/app/models/form/account_batch.rb b/app/models/form/account_batch.rb
index f1b7a4566..0b285fde9 100644
--- a/app/models/form/account_batch.rb
+++ b/app/models/form/account_batch.rb
@@ -69,6 +69,6 @@ class Form::AccountBatch
     records = accounts.includes(:user)
 
     records.each { |account| authorize(account.user, :reject?) }
-           .each { |account| SuspendAccountService.new.call(account, including_user: true, destroy: true, skip_distribution: true) }
+           .each { |account| SuspendAccountService.new.call(account, reserve_email: false, reserve_username: false) }
   end
 end
diff --git a/app/models/form/admin_settings.rb b/app/models/form/admin_settings.rb
index 57dd3edd9..f1ee38325 100644
--- a/app/models/form/admin_settings.rb
+++ b/app/models/form/admin_settings.rb
@@ -38,6 +38,7 @@ class Form::AdminSettings
     trends
     show_domain_blocks
     show_domain_blocks_rationale
+    noindex
   ).freeze
 
   BOOLEAN_KEYS = %i(
@@ -55,6 +56,7 @@ class Form::AdminSettings
     show_replies_in_public_timelines
     spam_check_enabled
     trends
+    noindex
   ).freeze
 
   UPLOAD_KEYS = %i(
diff --git a/app/models/form/custom_emoji_batch.rb b/app/models/form/custom_emoji_batch.rb
new file mode 100644
index 000000000..076e8c9e3
--- /dev/null
+++ b/app/models/form/custom_emoji_batch.rb
@@ -0,0 +1,106 @@
+# frozen_string_literal: true
+
+class Form::CustomEmojiBatch
+  include ActiveModel::Model
+  include Authorization
+  include AccountableConcern
+
+  attr_accessor :custom_emoji_ids, :action, :current_account,
+                :category_id, :category_name, :visible_in_picker
+
+  def save
+    case action
+    when 'update'
+      update!
+    when 'list'
+      list!
+    when 'unlist'
+      unlist!
+    when 'enable'
+      enable!
+    when 'disable'
+      disable!
+    when 'copy'
+      copy!
+    when 'delete'
+      delete!
+    end
+  end
+
+  private
+
+  def custom_emojis
+    CustomEmoji.where(id: custom_emoji_ids)
+  end
+
+  def update!
+    custom_emojis.each { |custom_emoji| authorize(custom_emoji, :update?) }
+
+    category = begin
+      if category_id.present?
+        CustomEmojiCategory.find(category_id)
+      elsif category_name.present?
+        CustomEmojiCategory.create!(name: category_name)
+      end
+    end
+
+    custom_emojis.each do |custom_emoji|
+      custom_emoji.update(category_id: category&.id)
+      log_action :update, custom_emoji
+    end
+  end
+
+  def list!
+    custom_emojis.each { |custom_emoji| authorize(custom_emoji, :update?) }
+
+    custom_emojis.each do |custom_emoji|
+      custom_emoji.update(visible_in_picker: true)
+      log_action :update, custom_emoji
+    end
+  end
+
+  def unlist!
+    custom_emojis.each { |custom_emoji| authorize(custom_emoji, :update?) }
+
+    custom_emojis.each do |custom_emoji|
+      custom_emoji.update(visible_in_picker: false)
+      log_action :update, custom_emoji
+    end
+  end
+
+  def enable!
+    custom_emojis.each { |custom_emoji| authorize(custom_emoji, :enable?) }
+
+    custom_emojis.each do |custom_emoji|
+      custom_emoji.update(disabled: false)
+      log_action :enable, custom_emoji
+    end
+  end
+
+  def disable!
+    custom_emojis.each { |custom_emoji| authorize(custom_emoji, :disable?) }
+
+    custom_emojis.each do |custom_emoji|
+      custom_emoji.update(disabled: true)
+      log_action :disable, custom_emoji
+    end
+  end
+
+  def copy!
+    custom_emojis.each { |custom_emoji| authorize(custom_emoji, :copy?) }
+
+    custom_emojis.each do |custom_emoji|
+      copied_custom_emoji = custom_emoji.copy!
+      log_action :create, copied_custom_emoji
+    end
+  end
+
+  def delete!
+    custom_emojis.each { |custom_emoji| authorize(custom_emoji, :destroy?) }
+
+    custom_emojis.each do |custom_emoji|
+      custom_emoji.destroy
+      log_action :destroy, custom_emoji
+    end
+  end
+end
diff --git a/app/models/form/status_batch.rb b/app/models/form/status_batch.rb
index e09cc2594..c4943a7ea 100644
--- a/app/models/form/status_batch.rb
+++ b/app/models/form/status_batch.rb
@@ -35,7 +35,7 @@ class Form::StatusBatch
   def delete_statuses
     Status.where(id: status_ids).reorder(nil).find_each do |status|
       status.discard
-      RemovalWorker.perform_async(status.id, redraft: false)
+      RemovalWorker.perform_async(status.id, immediate: true)
       Tombstone.find_or_create_by(uri: status.uri, account: status.account, by_moderator: true)
       log_action :destroy, status
     end
diff --git a/app/models/form/tag_batch.rb b/app/models/form/tag_batch.rb
new file mode 100644
index 000000000..fd517a1a6
--- /dev/null
+++ b/app/models/form/tag_batch.rb
@@ -0,0 +1,33 @@
+# frozen_string_literal: true
+
+class Form::TagBatch
+  include ActiveModel::Model
+  include Authorization
+
+  attr_accessor :tag_ids, :action, :current_account
+
+  def save
+    case action
+    when 'approve'
+      approve!
+    when 'reject'
+      reject!
+    end
+  end
+
+  private
+
+  def tags
+    Tag.where(id: tag_ids)
+  end
+
+  def approve!
+    tags.each { |tag| authorize(tag, :update?) }
+    tags.update_all(trendable: true, reviewed_at: Time.now.utc)
+  end
+
+  def reject!
+    tags.each { |tag| authorize(tag, :update?) }
+    tags.update_all(trendable: false, reviewed_at: Time.now.utc)
+  end
+end
diff --git a/app/models/marker.rb b/app/models/marker.rb
new file mode 100644
index 000000000..a5bd2176a
--- /dev/null
+++ b/app/models/marker.rb
@@ -0,0 +1,23 @@
+# frozen_string_literal: true
+
+# == Schema Information
+#
+# Table name: markers
+#
+#  id           :bigint(8)        not null, primary key
+#  user_id      :bigint(8)
+#  timeline     :string           default(""), not null
+#  last_read_id :bigint(8)        default(0), not null
+#  lock_version :integer          default(0), not null
+#  created_at   :datetime         not null
+#  updated_at   :datetime         not null
+#
+
+class Marker < ApplicationRecord
+  TIMELINES = %w(home notifications).freeze
+
+  belongs_to :user
+
+  validates :timeline, :last_read_id, presence: true
+  validates :timeline, inclusion: { in: TIMELINES }
+end
diff --git a/app/models/media_attachment.rb b/app/models/media_attachment.rb
index 83d1858aa..cbe23810b 100644
--- a/app/models/media_attachment.rb
+++ b/app/models/media_attachment.rb
@@ -118,17 +118,18 @@ class MediaAttachment < ApplicationRecord
   validates_attachment_content_type :file, content_type: IMAGE_MIME_TYPES + VIDEO_MIME_TYPES + AUDIO_MIME_TYPES
   validates_attachment_size :file, less_than: IMAGE_LIMIT, unless: :larger_media_format?
   validates_attachment_size :file, less_than: VIDEO_LIMIT, if: :larger_media_format?
-  remotable_attachment :file, VIDEO_LIMIT
+  remotable_attachment :file, VIDEO_LIMIT, suppress_errors: false
 
   include Attachmentable
 
   validates :account, presence: true
-  validates :description, length: { maximum: 420 }, if: :local?
+  validates :description, length: { maximum: 1_500 }, if: :local?
 
   scope :attached,   -> { where.not(status_id: nil).or(where.not(scheduled_status_id: nil)) }
   scope :unattached, -> { where(status_id: nil, scheduled_status_id: nil) }
   scope :local,      -> { where(remote_url: '') }
   scope :remote,     -> { where.not(remote_url: '') }
+  scope :cached,     -> { remote.where.not(file_file_name: nil) }
 
   default_scope { order(id: :asc) }
 
diff --git a/app/models/preview_card.rb b/app/models/preview_card.rb
index a792b352b..9d6c1938a 100644
--- a/app/models/preview_card.rb
+++ b/app/models/preview_card.rb
@@ -43,6 +43,8 @@ class PreviewCard < ApplicationRecord
   validates_attachment_size :image, less_than: LIMIT
   remotable_attachment :image, LIMIT
 
+  scope :cached, -> { where.not(image_file_name: [nil, '']) }
+
   before_save :extract_dimensions, if: :link?
 
   def save_with_optional_image!
diff --git a/app/models/report.rb b/app/models/report.rb
index 1e707ff1c..fb2e040ee 100644
--- a/app/models/report.rb
+++ b/app/models/report.rb
@@ -59,6 +59,7 @@ class Report < ApplicationRecord
   end
 
   def resolve!(acting_account)
+    RemovalWorker.push_bulk(Status.with_discarded.discarded.where(id: status_ids).pluck(:id)) { |status_id| [status_id, { immediate: true }] }
     update!(action_taken: true, action_taken_by_account_id: acting_account.id)
   end
 
diff --git a/app/models/status.rb b/app/models/status.rb
index 757deea06..b0f610aa2 100644
--- a/app/models/status.rb
+++ b/app/models/status.rb
@@ -221,6 +221,10 @@ class Status < ApplicationRecord
     !sensitive? && with_media?
   end
 
+  def reported?
+    @reported ||= Report.where(target_account: account).unresolved.where('? = ANY(status_ids)', id).exists?
+  end
+
   def emojis
     return @emojis if defined?(@emojis)
 
diff --git a/app/models/tag.rb b/app/models/tag.rb
index 135e0a030..a6aed0d68 100644
--- a/app/models/tag.rb
+++ b/app/models/tag.rb
@@ -25,8 +25,9 @@ class Tag < ApplicationRecord
   has_many :featured_tags, dependent: :destroy, inverse_of: :tag
   has_one :account_tag_stat, dependent: :destroy
 
-  HASHTAG_NAME_RE = '([[:word:]_][[:word:]_·]*[[:alpha:]_·][[:word:]_·]*[[:word:]_])|([[:word:]_]*[[:alpha:]][[:word:]_]*)'
-  HASHTAG_RE = /(?:^|[^\/\)\w])#(#{HASHTAG_NAME_RE})/i
+  HASHTAG_SEPARATORS = "_\u00B7\u200c"
+  HASHTAG_NAME_RE    = "([[:word:]_][[:word:]#{HASHTAG_SEPARATORS}]*[[:alpha:]#{HASHTAG_SEPARATORS}][[:word:]#{HASHTAG_SEPARATORS}]*[[:word:]_])|([[:word:]_]*[[:alpha:]][[:word:]_]*)"
+  HASHTAG_RE         = /(?:^|[^\/\)\w])#(#{HASHTAG_NAME_RE})/i
 
   validates :name, presence: true, format: { with: /\A(#{HASHTAG_NAME_RE})\z/i }
   validate :validate_name_change, if: -> { !new_record? && name_changed? }
diff --git a/app/models/trending_tags.rb b/app/models/trending_tags.rb
index e1b92b175..8cdade42d 100644
--- a/app/models/trending_tags.rb
+++ b/app/models/trending_tags.rb
@@ -7,8 +7,8 @@ class TrendingTags
   THRESHOLD            = 5
   LIMIT                = 10
   REVIEW_THRESHOLD     = 3
-  MAX_SCORE_COOLDOWN   = 3.days.freeze
-  MAX_SCORE_HALFLIFE   = 6.hours.freeze
+  MAX_SCORE_COOLDOWN   = 2.days.freeze
+  MAX_SCORE_HALFLIFE   = 2.hours.freeze
 
   class << self
     include Redisable
@@ -83,6 +83,7 @@ class TrendingTags
       # Trim older items
 
       redis.zremrangebyrank(KEY, 0, -(LIMIT + 1))
+      redis.zremrangebyscore(KEY, '(0.3', '-inf')
     end
 
     def get(limit, filtered: true)
diff --git a/app/models/user.rb b/app/models/user.rb
index 341227d3e..792246b1d 100644
--- a/app/models/user.rb
+++ b/app/models/user.rb
@@ -74,6 +74,7 @@ class User < ApplicationRecord
   has_many :applications, class_name: 'Doorkeeper::Application', as: :owner
   has_many :backups, inverse_of: :user
   has_many :invites, inverse_of: :user
+  has_many :markers, inverse_of: :user, dependent: :destroy
 
   has_one :invite_request, class_name: 'UserInviteRequest', inverse_of: :user, dependent: :destroy
   accepts_nested_attributes_for :invite_request, reject_if: ->(attributes) { attributes['text'].blank? }
@@ -171,6 +172,10 @@ class User < ApplicationRecord
     confirmed? && approved? && !disabled? && !account.suspended?
   end
 
+  def unconfirmed_or_pending?
+    !(confirmed? && approved?)
+  end
+
   def inactive_message
     !approved? ? :pending : super
   end