Merge branch 'glitch' into production

9 files changed, 150 insertions, 54 deletions

diff --git a/app/models/account.rb b/app/models/account.rb
index 3370fbc5e..92e60f747 100644
--- a/app/models/account.rb
+++ b/app/models/account.rb
@@ -231,17 +231,7 @@ class Account < ApplicationRecord
   end
 
   def tags_as_strings=(tag_names)
-    tag_names.map! { |name| name.mb_chars.downcase.to_s }
-    tag_names.uniq!
-
-    # Existing hashtags
-    hashtags_map = Tag.where(name: tag_names).each_with_object({}) { |tag, h| h[tag.name] = tag }
-
-    # Initialize not yet existing hashtags
-    tag_names.each do |name|
-      next if hashtags_map.key?(name)
-      hashtags_map[name] = Tag.new(name: name)
-    end
+    hashtags_map = Tag.find_or_create_by_names(tag_names).each_with_object({}) { |tag, h| h[tag.name] = tag }
 
     # Remove hashtags that are to be deleted
     tags.each do |tag|
diff --git a/app/models/application_record.rb b/app/models/application_record.rb
index 83134d41a..c1b873da6 100644
--- a/app/models/application_record.rb
+++ b/app/models/application_record.rb
@@ -2,5 +2,16 @@
 
 class ApplicationRecord < ActiveRecord::Base
   self.abstract_class = true
+
   include Remotable
+
+  def boolean_with_default(key, default_value)
+    value = attributes[key]
+
+    if value.nil?
+      default_value
+    else
+      value
+    end
+  end
 end
diff --git a/app/models/domain_block.rb b/app/models/domain_block.rb
index 25d3b87ef..3f5b9f23e 100644
--- a/app/models/domain_block.rb
+++ b/app/models/domain_block.rb
@@ -3,13 +3,15 @@
 #
 # Table name: domain_blocks
 #
-#  id             :bigint(8)        not null, primary key
-#  domain         :string           default(""), not null
-#  created_at     :datetime         not null
-#  updated_at     :datetime         not null
-#  severity       :integer          default("silence")
-#  reject_media   :boolean          default(FALSE), not null
-#  reject_reports :boolean          default(FALSE), not null
+#  id              :bigint(8)        not null, primary key
+#  domain          :string           default(""), not null
+#  created_at      :datetime         not null
+#  updated_at      :datetime         not null
+#  severity        :integer          default("silence")
+#  reject_media    :boolean          default(FALSE), not null
+#  reject_reports  :boolean          default(FALSE), not null
+#  private_comment :text
+#  public_comment  :text
 #
 
 class DomainBlock < ApplicationRecord
diff --git a/app/models/featured_tag.rb b/app/models/featured_tag.rb
index d06ae26a8..e02ae0705 100644
--- a/app/models/featured_tag.rb
+++ b/app/models/featured_tag.rb
@@ -23,7 +23,7 @@ class FeaturedTag < ApplicationRecord
   validate :validate_featured_tags_limit, on: :create
 
   def name=(str)
-    self.tag = Tag.find_or_initialize_by(name: str.strip.delete('#').mb_chars.downcase.to_s)
+    self.tag = Tag.find_or_create_by_names(str.strip)&.first
   end
 
   def increment(timestamp)
diff --git a/app/models/form/admin_settings.rb b/app/models/form/admin_settings.rb
index ecaed44f6..2c3a7f13b 100644
--- a/app/models/form/admin_settings.rb
+++ b/app/models/form/admin_settings.rb
@@ -35,6 +35,7 @@ class Form::AdminSettings
     show_reblogs_in_public_timelines
     show_replies_in_public_timelines
     spam_check_enabled
+    trends
   ).freeze
 
   BOOLEAN_KEYS = %i(
@@ -51,6 +52,7 @@ class Form::AdminSettings
     show_reblogs_in_public_timelines
     show_replies_in_public_timelines
     spam_check_enabled
+    trends
   ).freeze
 
   UPLOAD_KEYS = %i(
diff --git a/app/models/remote_follow.rb b/app/models/remote_follow.rb
index 2537de36c..93df11724 100644
--- a/app/models/remote_follow.rb
+++ b/app/models/remote_follow.rb
@@ -2,19 +2,21 @@
 
 class RemoteFollow
   include ActiveModel::Validations
+  include RoutingHelper
 
   attr_accessor :acct, :addressable_template
 
   validates :acct, presence: true
 
-  def initialize(attrs = nil)
-    @acct = attrs[:acct].gsub(/\A@/, '').strip if !attrs.nil? && !attrs[:acct].nil?
+  def initialize(attrs = {})
+    @acct = normalize_acct(attrs[:acct])
   end
 
   def valid?
     return false unless super
 
-    populate_template
+    fetch_template!
+
     errors.empty?
   end
 
@@ -28,8 +30,30 @@ class RemoteFollow
 
   private
 
-  def populate_template
-    if acct.blank? || redirect_url_link.nil? || redirect_url_link.template.nil?
+  def normalize_acct(value)
+    return if value.blank?
+
+    username, domain = value.strip.gsub(/\A@/, '').split('@')
+
+    domain = begin
+      if TagManager.instance.local_domain?(domain)
+        nil
+      else
+        TagManager.instance.normalize_domain(domain)
+      end
+    end
+
+    [username, domain].compact.join('@')
+  end
+
+  def fetch_template!
+    return missing_resource if acct.blank?
+
+    _, domain = acct.split('@')
+
+    if domain.nil?
+      @addressable_template = Addressable::Template.new("#{authorize_interaction_url}?uri={uri}")
+    elsif redirect_url_link.nil? || redirect_url_link.template.nil?
       missing_resource_error
     else
       @addressable_template = Addressable::Template.new(redirect_uri_template)
@@ -45,7 +69,7 @@ class RemoteFollow
   end
 
   def acct_resource
-    @_acct_resource ||= Goldfinger.finger("acct:#{acct}")
+    @acct_resource ||= Goldfinger.finger("acct:#{acct}")
   rescue Goldfinger::Error, HTTP::ConnectionError
     nil
   end
diff --git a/app/models/tag.rb b/app/models/tag.rb
index 46e3a3ec0..1364d1dba 100644
--- a/app/models/tag.rb
+++ b/app/models/tag.rb
@@ -3,10 +3,16 @@
 #
 # Table name: tags
 #
-#  id         :bigint(8)        not null, primary key
-#  name       :string           default(""), not null
-#  created_at :datetime         not null
-#  updated_at :datetime         not null
+#  id                  :bigint(8)        not null, primary key
+#  name                :string           default(""), not null
+#  created_at          :datetime         not null
+#  updated_at          :datetime         not null
+#  score               :integer
+#  usable              :boolean
+#  trendable           :boolean
+#  listable            :boolean
+#  reviewed_at         :datetime
+#  requested_review_at :datetime
 #
 
 class Tag < ApplicationRecord
@@ -21,16 +27,19 @@ class Tag < ApplicationRecord
   HASHTAG_RE = /(?:^|[^\/\)\w])#(#{HASHTAG_NAME_RE})/i
 
   validates :name, presence: true, format: { with: /\A(#{HASHTAG_NAME_RE})\z/i }
+  validate :validate_name_change, if: -> { !new_record? && name_changed? }
 
-  scope :discoverable, -> { joins(:account_tag_stat).where(AccountTagStat.arel_table[:accounts_count].gt(0)).where(account_tag_stats: { hidden: false }).order(Arel.sql('account_tag_stats.accounts_count desc')) }
-  scope :hidden, -> { where(account_tag_stats: { hidden: true }) }
+  scope :reviewed, -> { where.not(reviewed_at: nil) }
+  scope :unreviewed, -> { where(reviewed_at: nil) }
+  scope :pending_review, -> { unreviewed.where.not(requested_review_at: nil) }
+  scope :usable, -> { where(usable: [true, nil]) }
+  scope :discoverable, -> { where(listable: [true, nil]).joins(:account_tag_stat).where(AccountTagStat.arel_table[:accounts_count].gt(0)).order(Arel.sql('account_tag_stats.accounts_count desc')) }
   scope :most_used, ->(account) { joins(:statuses).where(statuses: { account: account }).group(:id).order(Arel.sql('count(*) desc')) }
 
   delegate :accounts_count,
            :accounts_count=,
            :increment_count!,
            :decrement_count!,
-           :hidden?,
            to: :account_tag_stat
 
   after_save :save_account_tag_stat
@@ -47,6 +56,40 @@ class Tag < ApplicationRecord
     name
   end
 
+  def usable
+    boolean_with_default('usable', true)
+  end
+
+  alias usable? usable
+
+  def listable
+    boolean_with_default('listable', true)
+  end
+
+  alias listable? listable
+
+  def trendable
+    boolean_with_default('trendable', false)
+  end
+
+  alias trendable? trendable
+
+  def requires_review?
+    reviewed_at.nil?
+  end
+
+  def reviewed?
+    reviewed_at.present?
+  end
+
+  def requested_review?
+    requested_review_at.present?
+  end
+
+  def trending?
+    TrendingTags.trending?(self)
+  end
+
   def history
     days = []
 
@@ -75,10 +118,12 @@ class Tag < ApplicationRecord
     end
 
     def search_for(term, limit = 5, offset = 0)
-      pattern = sanitize_sql_like(normalize(term.strip)) + '%'
+      normalized_term = normalize(term.strip).mb_chars.downcase.to_s
+      pattern         = sanitize_sql_like(normalized_term) + '%'
 
-      Tag.where(arel_table[:name].lower.matches(pattern.mb_chars.downcase.to_s))
-         .order(:name)
+      Tag.where(arel_table[:name].lower.matches(pattern))
+         .where(arel_table[:score].gt(0).or(arel_table[:name].lower.eq(normalized_term)))
+         .order(Arel.sql('length(name) ASC, score DESC, name ASC'))
          .limit(limit)
          .offset(offset)
     end
@@ -114,4 +159,8 @@ class Tag < ApplicationRecord
     return unless account_tag_stat&.changed?
     account_tag_stat.save
   end
+
+  def validate_name_change
+    errors.add(:name, I18n.t('tags.does_not_match_previous_name')) unless name_was.mb_chars.casecmp(name.mb_chars).zero?
+  end
 end
diff --git a/app/models/trending_tags.rb b/app/models/trending_tags.rb
index 148535c21..594ae9520 100644
--- a/app/models/trending_tags.rb
+++ b/app/models/trending_tags.rb
@@ -5,23 +5,32 @@ class TrendingTags
   EXPIRE_HISTORY_AFTER = 7.days.seconds
   EXPIRE_TRENDS_AFTER  = 1.day.seconds
   THRESHOLD            = 5
+  LIMIT                = 10
 
   class << self
     include Redisable
 
     def record_use!(tag, account, at_time = Time.now.utc)
-      return if disallowed_hashtags.include?(tag.name) || account.silenced? || account.bot?
+      return if account.silenced? || account.bot? || !tag.usable? || !(tag.trendable? || tag.requires_review?)
 
       increment_historical_use!(tag.id, at_time)
       increment_unique_use!(tag.id, account.id, at_time)
-      increment_vote!(tag.id, at_time)
+      increment_vote!(tag, at_time)
     end
 
-    def get(limit)
-      key     = "#{KEY}:#{Time.now.utc.beginning_of_day.to_i}"
-      tag_ids = redis.zrevrange(key, 0, limit - 1).map(&:to_i)
-      tags    = Tag.where(id: tag_ids).to_a.each_with_object({}) { |tag, h| h[tag.id] = tag }
-      tag_ids.map { |tag_id| tags[tag_id] }.compact
+    def get(limit, filtered: true)
+      tag_ids = redis.zrevrange("#{KEY}:#{Time.now.utc.beginning_of_day.to_i}", 0, LIMIT - 1).map(&:to_i)
+
+      tags = Tag.where(id: tag_ids)
+      tags = tags.where(trendable: true) if filtered
+      tags = tags.each_with_object({}) { |tag, h| h[tag.id] = tag }
+
+      tag_ids.map { |tag_id| tags[tag_id] }.compact.take(limit)
+    end
+
+    def trending?(tag)
+      rank = redis.zrevrank("#{KEY}:#{Time.now.utc.beginning_of_day.to_i}", tag.id)
+      rank.present? && rank <= LIMIT
     end
 
     private
@@ -38,28 +47,31 @@ class TrendingTags
       redis.expire(key, EXPIRE_HISTORY_AFTER)
     end
 
-    def increment_vote!(tag_id, at_time)
+    def increment_vote!(tag, at_time)
       key      = "#{KEY}:#{at_time.beginning_of_day.to_i}"
-      expected = redis.pfcount("activity:tags:#{tag_id}:#{(at_time - 1.day).beginning_of_day.to_i}:accounts").to_f
+      expected = redis.pfcount("activity:tags:#{tag.id}:#{(at_time - 1.day).beginning_of_day.to_i}:accounts").to_f
       expected = 1.0 if expected.zero?
-      observed = redis.pfcount("activity:tags:#{tag_id}:#{at_time.beginning_of_day.to_i}:accounts").to_f
+      observed = redis.pfcount("activity:tags:#{tag.id}:#{at_time.beginning_of_day.to_i}:accounts").to_f
 
       if expected > observed || observed < THRESHOLD
-        redis.zrem(key, tag_id.to_s)
+        redis.zrem(key, tag.id)
       else
-        score = ((observed - expected)**2) / expected
-        redis.zadd(key, score, tag_id.to_s)
+        score    = ((observed - expected)**2) / expected
+        old_rank = redis.zrevrank(key, tag.id)
+
+        redis.zadd(key, score, tag.id)
+        request_review!(tag) if (old_rank.nil? || old_rank > LIMIT) && redis.zrevrank(key, tag.id) <= LIMIT && !tag.trendable? && tag.requires_review? && !tag.requested_review?
       end
 
       redis.expire(key, EXPIRE_TRENDS_AFTER)
     end
 
-    def disallowed_hashtags
-      return @disallowed_hashtags if defined?(@disallowed_hashtags)
+    def request_review!(tag)
+      return unless Setting.trends
+
+      tag.touch(:requested_review_at)
 
-      @disallowed_hashtags = Setting.disallowed_hashtags.nil? ? [] : Setting.disallowed_hashtags
-      @disallowed_hashtags = @disallowed_hashtags.split(' ') if @disallowed_hashtags.is_a? String
-      @disallowed_hashtags = @disallowed_hashtags.map(&:downcase)
+      User.staff.includes(:account).find_each { |u| AdminMailer.new_trending_tag(u.account, tag).deliver_later! if u.allows_trending_tag_emails? }
     end
   end
 end
diff --git a/app/models/user.rb b/app/models/user.rb
index 2a7fffca5..45a4b8989 100644
--- a/app/models/user.rb
+++ b/app/models/user.rb
@@ -107,7 +107,9 @@ class User < ApplicationRecord
   delegate :auto_play_gif, :default_sensitive, :unfollow_modal, :boost_modal, :favourite_modal, :delete_modal,
            :reduce_motion, :system_font_ui, :noindex, :flavour, :skin, :display_media, :hide_network, :hide_followers_count,
            :expand_spoilers, :default_language, :aggregate_reblogs, :show_application,
-           :advanced_layout, :default_content_type, :use_blurhash, :use_pending_items, :use_pending_items, to: :settings, prefix: :setting, allow_nil: false
+           :advanced_layout, :use_blurhash, :use_pending_items, :trends,
+           :default_content_type,
+           to: :settings, prefix: :setting, allow_nil: false
 
   attr_reader :invite_code
   attr_writer :external
@@ -207,6 +209,10 @@ class User < ApplicationRecord
     settings.notification_emails['pending_account']
   end
 
+  def allows_trending_tag_emails?
+    settings.notification_emails['trending_tag']
+  end
+
   def hides_network?
     @hides_network ||= settings.hide_network
   end