From 115dab78f1cc5357281dcb593f04ac8b2629cec6 Mon Sep 17 00:00:00 2001
From: Eugen Rochko <eugen@zeonfederated.com>
Date: Mon, 5 Aug 2019 19:54:29 +0200
Subject: Change admin UI for hashtags and add back whitelisted trends (#11490)

Fix #271

Add back the `GET /api/v1/trends` API with the caveat that it does
not return tags that have not been allowed to trend by the staff.

When a hashtag begins to trend (internally) and that hashtag has
not been previously reviewed by the staff, the staff is notified.

The new admin UI for hashtags allows filtering hashtags by where
they are used (e.g. in the profile directory), whether they have
been reviewed or are pending reviewal, they show by how many people
the hashtag is used in the directory, how many people used it
today, how many statuses with it have been created today, and it
allows fixing the name of the hashtag to make it more readable.

The disallowed hashtags feature has been reworked. It is now
controlled from the admin UI for hashtags instead of from
the file `config/settings.yml`
---
 app/models/trending_tags.rb | 48 +++++++++++++++++++++++----------------------
 1 file changed, 25 insertions(+), 23 deletions(-)

(limited to 'app/models/trending_tags.rb')

diff --git a/app/models/trending_tags.rb b/app/models/trending_tags.rb
index 211c8f1dc..e9b9b25e3 100644
--- a/app/models/trending_tags.rb
+++ b/app/models/trending_tags.rb
@@ -10,20 +10,28 @@ class TrendingTags
     include Redisable
 
     def record_use!(tag, account, at_time = Time.now.utc)
-      return if disallowed_hashtags.include?(tag.name) || account.silenced? || account.bot?
+      return if account.silenced? || account.bot? || !tag.usable? || !(tag.trendable? || tag.requires_review?)
 
       increment_historical_use!(tag.id, at_time)
       increment_unique_use!(tag.id, account.id, at_time)
-      increment_vote!(tag.id, at_time)
+      increment_vote!(tag, at_time)
     end
 
-    def get(limit)
-      key     = "#{KEY}:#{Time.now.utc.beginning_of_day.to_i}"
-      tag_ids = redis.zrevrange(key, 0, limit - 1).map(&:to_i)
-      tags    = Tag.where(id: tag_ids).to_a.each_with_object({}) { |tag, h| h[tag.id] = tag }
+    def get(limit, filtered: true)
+      tag_ids = redis.zrevrange("#{KEY}:#{Time.now.utc.beginning_of_day.to_i}", 0, limit - 1).map(&:to_i)
+
+      tags = Tag.where(id: tag_ids)
+      tags = tags.where(trendable: true) if filtered
+      tags = tags.each_with_object({}) { |tag, h| h[tag.id] = tag }
+
       tag_ids.map { |tag_id| tags[tag_id] }.compact
     end
 
+    def trending?(tag)
+      rank = redis.zrevrank("#{KEY}:#{Time.now.utc.beginning_of_day.to_i}", tag.id)
+      rank.present? && rank <= 10
+    end
+
     private
 
     def increment_historical_use!(tag_id, at_time)
@@ -38,33 +46,27 @@ class TrendingTags
       redis.expire(key, EXPIRE_HISTORY_AFTER)
     end
 
-    def increment_vote!(tag_id, at_time)
+    def increment_vote!(tag, at_time)
       key      = "#{KEY}:#{at_time.beginning_of_day.to_i}"
-      expected = redis.pfcount("activity:tags:#{tag_id}:#{(at_time - 1.day).beginning_of_day.to_i}:accounts").to_f
+      expected = redis.pfcount("activity:tags:#{tag.id}:#{(at_time - 1.day).beginning_of_day.to_i}:accounts").to_f
       expected = 1.0 if expected.zero?
-      observed = redis.pfcount("activity:tags:#{tag_id}:#{at_time.beginning_of_day.to_i}:accounts").to_f
+      observed = redis.pfcount("activity:tags:#{tag.id}:#{at_time.beginning_of_day.to_i}:accounts").to_f
 
       if expected > observed || observed < THRESHOLD
-        redis.zrem(key, tag_id.to_s)
+        redis.zrem(key, tag.id)
       else
-        score = ((observed - expected)**2) / expected
-        added = redis.zadd(key, score, tag_id.to_s)
-        bump_tag_score!(tag_id) if added
+        score    = ((observed - expected)**2) / expected
+        old_rank = redis.zrevrank(key, tag.id)
+
+        redis.zadd(key, score, tag.id)
+        request_review!(tag) if (old_rank.nil? || old_rank > 10) && redis.zrevrank(key, tag.id) <= 10 && !tag.trendable? && tag.requires_review? && !tag.requested_review?
       end
 
       redis.expire(key, EXPIRE_TRENDS_AFTER)
     end
 
-    def bump_tag_score!(tag_id)
-      Tag.where(id: tag_id).update_all('score = COALESCE(score, 0) + 1')
-    end
-
-    def disallowed_hashtags
-      return @disallowed_hashtags if defined?(@disallowed_hashtags)
-
-      @disallowed_hashtags = Setting.disallowed_hashtags.nil? ? [] : Setting.disallowed_hashtags
-      @disallowed_hashtags = @disallowed_hashtags.split(' ') if @disallowed_hashtags.is_a? String
-      @disallowed_hashtags = @disallowed_hashtags.map(&:downcase)
+    def request_review!(tag)
+      User.staff.includes(:account).find_each { |u| AdminMailer.new_trending_tag(u.account, tag).deliver_later! if u.allows_trending_tag_emails? }
     end
   end
 end
-- 
cgit