Merge branch 'master' into glitch-soc/merge-upstream

Conflicts:
- app/models/status.rb

Resolved by taking both changes (not a real conflict, just changes too close
to each other).

13 files changed, 139 insertions, 56 deletions

diff --git a/app/models/account.rb b/app/models/account.rb
index bd3dc9c96..645a303c3 100644
--- a/app/models/account.rb
+++ b/app/models/account.rb
@@ -32,9 +32,6 @@
 #  suspended               :boolean          default(FALSE), not null
 #  locked                  :boolean          default(FALSE), not null
 #  header_remote_url       :string           default(""), not null
-#  statuses_count          :integer          default(0), not null
-#  followers_count         :integer          default(0), not null
-#  following_count         :integer          default(0), not null
 #  last_webfingered_at     :datetime
 #  inbox_url               :string           default(""), not null
 #  outbox_url              :string           default(""), not null
@@ -49,7 +46,7 @@
 #
 
 class Account < ApplicationRecord
-  USERNAME_RE = /[a-z0-9_]+([a-z0-9_\.]+[a-z0-9_]+)?/i
+  USERNAME_RE = /[a-z0-9_]+([a-z0-9_\.-]+[a-z0-9_]+)?/i
   MENTION_RE  = /(?<=^|[^\/[:word:]])@((#{USERNAME_RE})(?:@[a-z0-9\.\-]+[a-z0-9]+)?)/i
 
   include AccountAvatar
@@ -58,6 +55,7 @@ class Account < ApplicationRecord
   include AccountInteractions
   include Attachmentable
   include Paginable
+  include AccountCounters
 
   MAX_DISPLAY_NAME_LENGTH = (ENV['MAX_DISPLAY_NAME_CHARS'] || 30).to_i
   MAX_NOTE_LENGTH = (ENV['MAX_BIO_CHARS'] || 500).to_i
@@ -124,14 +122,13 @@ class Account < ApplicationRecord
 
   scope :remote, -> { where.not(domain: nil) }
   scope :local, -> { where(domain: nil) }
-  scope :without_followers, -> { where(followers_count: 0) }
-  scope :with_followers, -> { where('followers_count > 0') }
   scope :expiring, ->(time) { remote.where.not(subscription_expires_at: nil).where('subscription_expires_at < ?', time) }
   scope :partitioned, -> { order(Arel.sql('row_number() over (partition by domain)')) }
   scope :silenced, -> { where(silenced: true) }
   scope :suspended, -> { where(suspended: true) }
   scope :without_suspended, -> { where(suspended: false) }
   scope :recent, -> { reorder(id: :desc) }
+  scope :bots, -> { where(actor_type: %w(Application Service)) }
   scope :alphabetic, -> { order(domain: :asc, username: :asc) }
   scope :by_domain_accounts, -> { group(:domain).select(:domain, 'COUNT(*) AS accounts_count').order('accounts_count desc') }
   scope :matches_username, ->(value) { where(arel_table[:username].matches("#{value}%")) }
@@ -388,7 +385,9 @@ class Account < ApplicationRecord
         LIMIT ?
       SQL
 
-      find_by_sql([sql, limit])
+      records = find_by_sql([sql, limit])
+      ActiveRecord::Associations::Preloader.new.preload(records, :account_stat)
+      records
     end
 
     def advanced_search_for(terms, account, limit = 10, following = false)
@@ -415,7 +414,7 @@ class Account < ApplicationRecord
           LIMIT ?
         SQL
 
-        find_by_sql([sql, account.id, account.id, account.id, limit])
+        records = find_by_sql([sql, account.id, account.id, account.id, limit])
       else
         sql = <<-SQL.squish
           SELECT
@@ -431,8 +430,11 @@ class Account < ApplicationRecord
           LIMIT ?
         SQL
 
-        find_by_sql([sql, account.id, account.id, limit])
+        records = find_by_sql([sql, account.id, account.id, limit])
       end
+
+      ActiveRecord::Associations::Preloader.new.preload(records, :account_stat)
+      records
     end
 
     private
diff --git a/app/models/account_filter.rb b/app/models/account_filter.rb
index 84364bf1b..b10f50db7 100644
--- a/app/models/account_filter.rb
+++ b/app/models/account_filter.rb
@@ -5,13 +5,14 @@ class AccountFilter
 
   def initialize(params)
     @params = params
+    set_defaults!
   end
 
   def results
-    scope = Account.recent
+    scope = Account.recent.includes(:user)
 
     params.each do |key, value|
-      scope.merge!(scope_for(key, value)) if value.present?
+      scope.merge!(scope_for(key, value.to_s.strip)) if value.present?
     end
 
     scope
@@ -19,6 +20,11 @@ class AccountFilter
 
   private
 
+  def set_defaults!
+    params['local']  = '1' if params['remote'].blank?
+    params['active'] = '1' if params['suspended'].blank? && params['silenced'].blank?
+  end
+
   def scope_for(key, value)
     case key.to_s
     when 'local'
@@ -27,10 +33,10 @@ class AccountFilter
       Account.remote
     when 'by_domain'
       Account.where(domain: value)
+    when 'active'
+      Account.without_suspended
     when 'silenced'
       Account.silenced
-    when 'alphabetic'
-      Account.reorder(nil).alphabetic
     when 'suspended'
       Account.suspended
     when 'username'
@@ -40,11 +46,7 @@ class AccountFilter
     when 'email'
       accounts_with_users.merge User.matches_email(value)
     when 'ip'
-      if valid_ip?(value)
-        accounts_with_users.merge User.with_recent_ip_address(value)
-      else
-        Account.default_scoped
-      end
+      valid_ip?(value) ? accounts_with_users.where('users.current_sign_in_ip <<= ?', value) : Account.none
     when 'staff'
       accounts_with_users.merge User.staff
     else
@@ -57,8 +59,7 @@ class AccountFilter
   end
 
   def valid_ip?(value)
-    IPAddr.new(value)
-    true
+    IPAddr.new(value) && true
   rescue IPAddr::InvalidAddressError
     false
   end
diff --git a/app/models/account_stat.rb b/app/models/account_stat.rb
new file mode 100644
index 000000000..d5715268e
--- /dev/null
+++ b/app/models/account_stat.rb
@@ -0,0 +1,26 @@
+# frozen_string_literal: true
+
+# == Schema Information
+#
+# Table name: account_stats
+#
+#  id              :bigint(8)        not null, primary key
+#  account_id      :bigint(8)        not null
+#  statuses_count  :bigint(8)        default(0), not null
+#  following_count :bigint(8)        default(0), not null
+#  followers_count :bigint(8)        default(0), not null
+#  created_at      :datetime         not null
+#  updated_at      :datetime         not null
+#
+
+class AccountStat < ApplicationRecord
+  belongs_to :account, inverse_of: :account_stat
+
+  def increment_count!(key)
+    update(key => public_send(key) + 1)
+  end
+
+  def decrement_count!(key)
+    update(key => [public_send(key) - 1, 0].max)
+  end
+end
diff --git a/app/models/concerns/account_counters.rb b/app/models/concerns/account_counters.rb
new file mode 100644
index 000000000..fa3ec9a3d
--- /dev/null
+++ b/app/models/concerns/account_counters.rb
@@ -0,0 +1,31 @@
+# frozen_string_literal: true
+
+module AccountCounters
+  extend ActiveSupport::Concern
+
+  included do
+    has_one :account_stat, inverse_of: :account
+    after_save :save_account_stat
+  end
+
+  delegate :statuses_count,
+           :statuses_count=,
+           :following_count,
+           :following_count=,
+           :followers_count,
+           :followers_count=,
+           :increment_count!,
+           :decrement_count!,
+           to: :account_stat
+
+  def account_stat
+    super || build_account_stat
+  end
+
+  private
+
+  def save_account_stat
+    return unless account_stat&.changed?
+    account_stat.save
+  end
+end
diff --git a/app/models/concerns/account_interactions.rb b/app/models/concerns/account_interactions.rb
index ff57a884b..f27d39483 100644
--- a/app/models/concerns/account_interactions.rb
+++ b/app/models/concerns/account_interactions.rb
@@ -45,9 +45,9 @@ module AccountInteractions
     end
 
     def domain_blocking_map(target_account_ids, account_id)
-      accounts_map    = Account.where(id: target_account_ids).select('id, domain').map { |a| [a.id, a.domain] }.to_h
+      accounts_map    = Account.where(id: target_account_ids).select('id, domain').each_with_object({}) { |a, h| h[a.id] = a.domain }
       blocked_domains = domain_blocking_map_by_domain(accounts_map.values.compact, account_id)
-      accounts_map.map { |id, domain| [id, blocked_domains[domain]] }.to_h
+      accounts_map.reduce({}) { |h, (id, domain)| h.merge(id => blocked_domains[domain]) }
     end
 
     def domain_blocking_map_by_domain(target_domains, account_id)
diff --git a/app/models/concerns/status_threading_concern.rb b/app/models/concerns/status_threading_concern.rb
index fa441469c..b9c800c2a 100644
--- a/app/models/concerns/status_threading_concern.rb
+++ b/app/models/concerns/status_threading_concern.rb
@@ -8,7 +8,7 @@ module StatusThreadingConcern
   end
 
   def descendants(limit, account = nil, max_child_id = nil, since_child_id = nil, depth = nil)
-    find_statuses_from_tree_path(descendant_ids(limit, max_child_id, since_child_id, depth), account)
+    find_statuses_from_tree_path(descendant_ids(limit, max_child_id, since_child_id, depth), account, promote: true)
   end
 
   private
@@ -76,7 +76,7 @@ module StatusThreadingConcern
     descendants_with_self - [self]
   end
 
-  def find_statuses_from_tree_path(ids, account)
+  def find_statuses_from_tree_path(ids, account, promote: false)
     statuses    = statuses_with_accounts(ids).to_a
     account_ids = statuses.map(&:account_id).uniq
     domains     = statuses.map(&:account_domain).compact.uniq
@@ -86,6 +86,28 @@ module StatusThreadingConcern
 
     # Order ancestors/descendants by tree path
     statuses.sort_by! { |status| ids.index(status.id) }
+
+    # Bring self-replies to the top
+    if promote
+      promote_by!(statuses) { |status| status.in_reply_to_account_id == status.account_id }
+    else
+      statuses
+    end
+  end
+
+  def promote_by!(arr)
+    insert_at = arr.find_index { |item| !yield(item) }
+
+    return arr if insert_at.nil?
+
+    arr.each_with_index do |item, index|
+      next if index <= insert_at || !yield(item)
+
+      arr.insert(insert_at, arr.delete_at(index))
+      insert_at += 1
+    end
+
+    arr
   end
 
   def relations_map_for_account(account, account_ids, domains)
diff --git a/app/models/follow.rb b/app/models/follow.rb
index 7ad56eb78..87fa11425 100644
--- a/app/models/follow.rb
+++ b/app/models/follow.rb
@@ -16,11 +16,8 @@ class Follow < ApplicationRecord
   include Paginable
   include RelationshipCacheable
 
-  belongs_to :account, counter_cache: :following_count
-
-  belongs_to :target_account,
-             class_name: 'Account',
-             counter_cache: :followers_count
+  belongs_to :account
+  belongs_to :target_account, class_name: 'Account'
 
   has_one :notification, as: :activity, dependent: :destroy
 
@@ -39,7 +36,9 @@ class Follow < ApplicationRecord
   end
 
   before_validation :set_uri, only: :create
+  after_create :increment_cache_counters
   after_destroy :remove_endorsements
+  after_destroy :decrement_cache_counters
 
   private
 
@@ -50,4 +49,14 @@ class Follow < ApplicationRecord
   def remove_endorsements
     AccountPin.where(target_account_id: target_account_id, account_id: account_id).delete_all
   end
+
+  def increment_cache_counters
+    account&.increment_count!(:following_count)
+    target_account&.increment_count!(:followers_count)
+  end
+
+  def decrement_cache_counters
+    account&.decrement_count!(:following_count)
+    target_account&.decrement_count!(:followers_count)
+  end
 end
diff --git a/app/models/media_attachment.rb b/app/models/media_attachment.rb
index 0f787ebc4..a034b55fd 100644
--- a/app/models/media_attachment.rb
+++ b/app/models/media_attachment.rb
@@ -77,6 +77,7 @@ class MediaAttachment < ApplicationRecord
     format: 'mp4',
     convert_options: {
       output: {
+        'loglevel' => 'fatal',
         'movflags' => 'faststart',
         'pix_fmt'  => 'yuv420p',
         'vf'       => 'scale=\'trunc(iw/2)*2:trunc(ih/2)*2\'',
diff --git a/app/models/notification.rb b/app/models/notification.rb
index 78b180301..2f0a9b78c 100644
--- a/app/models/notification.rb
+++ b/app/models/notification.rb
@@ -75,7 +75,7 @@ class Notification < ApplicationRecord
 
       return if account_ids.empty?
 
-      accounts = Account.where(id: account_ids).map { |a| [a.id, a] }.to_h
+      accounts = Account.where(id: account_ids).includes(:account_stat).each_with_object({}) { |a, h| h[a.id] = a }
 
       cached_items.each do |item|
         item.from_account = accounts[item.from_account_id]
diff --git a/app/models/setting.rb b/app/models/setting.rb
index 033d09fd5..a5878e96a 100644
--- a/app/models/setting.rb
+++ b/app/models/setting.rb
@@ -40,7 +40,7 @@ class Setting < RailsSettings::Base
 
     def all_as_records
       vars    = thing_scoped
-      records = vars.map { |r| [r.var, r] }.to_h
+      records = vars.each_with_object({}) { |r, h| h[r.var] = r }
 
       default_settings.each do |key, default_value|
         next if records.key?(key) || default_value.is_a?(Hash)
diff --git a/app/models/status.rb b/app/models/status.rb
index e73f11503..65cd97c51 100644
--- a/app/models/status.rb
+++ b/app/models/status.rb
@@ -99,17 +99,16 @@ class Status < ApplicationRecord
 
   scope :not_local_only, -> { where(local_only: [false, nil]) }
 
-  cache_associated :account,
-                   :application,
+  cache_associated :application,
                    :media_attachments,
                    :conversation,
                    :status_stat,
                    :tags,
                    :preview_cards,
                    :stream_entry,
-                   active_mentions: :account,
+                   account: :account_stat,
+                   active_mentions: { account: :account_stat },
                    reblog: [
-                     :account,
                      :application,
                      :stream_entry,
                      :tags,
@@ -117,9 +116,10 @@ class Status < ApplicationRecord
                      :media_attachments,
                      :conversation,
                      :status_stat,
-                     active_mentions: :account,
+                     account: :account_stat,
+                     active_mentions: { account: :account_stat },
                    ],
-                   thread: :account
+                   thread: { account: :account_stat }
 
   delegate :domain, to: :account, prefix: true
 
@@ -328,7 +328,7 @@ class Status < ApplicationRecord
     end
 
     def favourites_map(status_ids, account_id)
-      Favourite.select('status_id').where(status_id: status_ids).where(account_id: account_id).map { |f| [f.status_id, true] }.to_h
+      Favourite.select('status_id').where(status_id: status_ids).where(account_id: account_id).each_with_object({}) { |f, h| h[f.status_id] = true }
     end
 
     def bookmarks_map(status_ids, account_id)
@@ -336,15 +336,15 @@ class Status < ApplicationRecord
     end
 
     def reblogs_map(status_ids, account_id)
-      select('reblog_of_id').where(reblog_of_id: status_ids).where(account_id: account_id).reorder(nil).map { |s| [s.reblog_of_id, true] }.to_h
+      select('reblog_of_id').where(reblog_of_id: status_ids).where(account_id: account_id).reorder(nil).each_with_object({}) { |s, h| h[s.reblog_of_id] = true }
     end
 
     def mutes_map(conversation_ids, account_id)
-      ConversationMute.select('conversation_id').where(conversation_id: conversation_ids).where(account_id: account_id).map { |m| [m.conversation_id, true] }.to_h
+      ConversationMute.select('conversation_id').where(conversation_id: conversation_ids).where(account_id: account_id).each_with_object({}) { |m, h| h[m.conversation_id] = true }
     end
 
     def pins_map(status_ids, account_id)
-      StatusPin.select('status_id').where(status_id: status_ids).where(account_id: account_id).map { |p| [p.status_id, true] }.to_h
+      StatusPin.select('status_id').where(status_id: status_ids).where(account_id: account_id).each_with_object({}) { |p, h| h[p.status_id] = true }
     end
 
     def reload_stale_associations!(cached_items)
@@ -359,7 +359,7 @@ class Status < ApplicationRecord
 
       return if account_ids.empty?
 
-      accounts = Account.where(id: account_ids).map { |a| [a.id, a] }.to_h
+      accounts = Account.where(id: account_ids).includes(:account_stat).each_with_object({}) { |a, h| h[a.id] = a }
 
       cached_items.each do |item|
         item.account = accounts[item.account_id]
@@ -471,6 +471,8 @@ class Status < ApplicationRecord
   end
 
   def set_conversation
+    self.thread = thread.reblog if thread&.reblog?
+
     self.reply = !(in_reply_to_id.nil? && thread.nil?) unless reply
 
     if reply? && !thread.nil?
@@ -501,12 +503,7 @@ class Status < ApplicationRecord
   def increment_counter_caches
     return if direct_visibility?
 
-    if association(:account).loaded?
-      account.update_attribute(:statuses_count, account.statuses_count + 1)
-    else
-      Account.where(id: account_id).update_all('statuses_count = COALESCE(statuses_count, 0) + 1')
-    end
-
+    account&.increment_count!(:statuses_count)
     reblog&.increment_count!(:reblogs_count) if reblog?
     thread&.increment_count!(:replies_count) if in_reply_to_id.present? && (public_visibility? || unlisted_visibility?)
   end
@@ -514,12 +511,7 @@ class Status < ApplicationRecord
   def decrement_counter_caches
     return if direct_visibility? || marked_for_mass_destruction?
 
-    if association(:account).loaded?
-      account.update_attribute(:statuses_count, [account.statuses_count - 1, 0].max)
-    else
-      Account.where(id: account_id).update_all('statuses_count = GREATEST(COALESCE(statuses_count, 0) - 1, 0)')
-    end
-
+    account&.decrement_count!(:statuses_count)
     reblog&.decrement_count!(:reblogs_count) if reblog?
     thread&.decrement_count!(:replies_count) if in_reply_to_id.present? && (public_visibility? || unlisted_visibility?)
   end
diff --git a/app/models/trending_tags.rb b/app/models/trending_tags.rb
index c559651c6..3a8be2164 100644
--- a/app/models/trending_tags.rb
+++ b/app/models/trending_tags.rb
@@ -18,7 +18,7 @@ class TrendingTags
     def get(limit)
       key     = "#{KEY}:#{Time.now.utc.beginning_of_day.to_i}"
       tag_ids = redis.zrevrange(key, 0, limit - 1).map(&:to_i)
-      tags    = Tag.where(id: tag_ids).to_a.map { |tag| [tag.id, tag] }.to_h
+      tags    = Tag.where(id: tag_ids).to_a.each_with_object({}) { |tag, h| h[tag.id] = tag }
       tag_ids.map { |tag_id| tags[tag_id] }.compact
     end
 
diff --git a/app/models/user.rb b/app/models/user.rb
index b9e18eecd..704523d34 100644
--- a/app/models/user.rb
+++ b/app/models/user.rb
@@ -83,7 +83,6 @@ class User < ApplicationRecord
   scope :inactive, -> { where(arel_table[:current_sign_in_at].lt(ACTIVE_DURATION.ago)) }
   scope :active, -> { confirmed.where(arel_table[:current_sign_in_at].gteq(ACTIVE_DURATION.ago)).joins(:account).where(accounts: { suspended: false }) }
   scope :matches_email, ->(value) { where(arel_table[:email].matches("#{value}%")) }
-  scope :with_recent_ip_address, ->(value) { where(arel_table[:current_sign_in_ip].eq(value).or(arel_table[:last_sign_in_ip].eq(value))) }
 
   before_validation :sanitize_languages