From 6e50134a42cb303e6e42f89f9ddb5aacf83e7a6d Mon Sep 17 00:00:00 2001 From: Eugen Rochko Date: Thu, 25 Nov 2021 13:07:38 +0100 Subject: Add trending links (#16917) * Add trending links * Add overriding specific links trendability * Add link type to preview cards and only trend articles Change trends review notifications from being sent every 5 minutes to being sent every 2 hours Change threshold from 5 unique accounts to 15 unique accounts * Fix tests --- app/models/account_statuses_cleanup_policy.rb | 4 +- app/models/form/preview_card_batch.rb | 65 +++++++++++++ app/models/form/preview_card_provider_batch.rb | 33 +++++++ app/models/form/tag_batch.rb | 8 +- app/models/preview_card.rb | 42 +++++++- app/models/preview_card_filter.rb | 53 ++++++++++ app/models/preview_card_provider.rb | 57 +++++++++++ app/models/preview_card_provider_filter.rb | 49 ++++++++++ app/models/tag.rb | 23 +---- app/models/tag_filter.rb | 56 +++++++---- app/models/trending_tags.rb | 128 ------------------------- app/models/trends.rb | 27 ++++++ app/models/trends/base.rb | 80 ++++++++++++++++ app/models/trends/history.rb | 98 +++++++++++++++++++ app/models/trends/links.rb | 117 ++++++++++++++++++++++ app/models/trends/tags.rb | 111 +++++++++++++++++++++ 16 files changed, 775 insertions(+), 176 deletions(-) create mode 100644 app/models/form/preview_card_batch.rb create mode 100644 app/models/form/preview_card_provider_batch.rb create mode 100644 app/models/preview_card_filter.rb create mode 100644 app/models/preview_card_provider.rb create mode 100644 app/models/preview_card_provider_filter.rb delete mode 100644 app/models/trending_tags.rb create mode 100644 app/models/trends.rb create mode 100644 app/models/trends/base.rb create mode 100644 app/models/trends/history.rb create mode 100644 app/models/trends/links.rb create mode 100644 app/models/trends/tags.rb (limited to 'app/models') diff --git a/app/models/account_statuses_cleanup_policy.rb b/app/models/account_statuses_cleanup_policy.rb index 0a9551ec2..0f78c1a54 100644 --- a/app/models/account_statuses_cleanup_policy.rb +++ b/app/models/account_statuses_cleanup_policy.rb @@ -4,8 +4,8 @@ # # Table name: account_statuses_cleanup_policies # -# id :bigint not null, primary key -# account_id :bigint not null +# id :bigint(8) not null, primary key +# account_id :bigint(8) not null # enabled :boolean default(TRUE), not null # min_status_age :integer default(1209600), not null # keep_direct :boolean default(TRUE), not null diff --git a/app/models/form/preview_card_batch.rb b/app/models/form/preview_card_batch.rb new file mode 100644 index 000000000..5f6e6522a --- /dev/null +++ b/app/models/form/preview_card_batch.rb @@ -0,0 +1,65 @@ +# frozen_string_literal: true + +class Form::PreviewCardBatch + include ActiveModel::Model + include Authorization + + attr_accessor :preview_card_ids, :action, :current_account, :precision + + def save + case action + when 'approve' + approve! + when 'approve_all' + approve_all! + when 'reject' + reject! + when 'reject_all' + reject_all! + end + end + + private + + def preview_cards + @preview_cards ||= PreviewCard.where(id: preview_card_ids) + end + + def preview_card_providers + @preview_card_providers ||= preview_cards.map(&:domain).uniq.map { |domain| PreviewCardProvider.matching_domain(domain) || PreviewCardProvider.new(domain: domain) } + end + + def approve! + preview_cards.each { |preview_card| authorize(preview_card, :update?) } + preview_cards.update_all(trendable: true) + end + + def approve_all! + preview_card_providers.each do |provider| + authorize(provider, :update?) + provider.update(trendable: true, reviewed_at: action_time) + end + + # Reset any individual overrides + preview_cards.update_all(trendable: nil) + end + + def reject! + preview_cards.each { |preview_card| authorize(preview_card, :update?) } + preview_cards.update_all(trendable: false) + end + + def reject_all! + preview_card_providers.each do |provider| + authorize(provider, :update?) + provider.update(trendable: false, reviewed_at: action_time) + end + + # Reset any individual overrides + preview_cards.update_all(trendable: nil) + end + + def action_time + @action_time ||= Time.now.utc + end +end diff --git a/app/models/form/preview_card_provider_batch.rb b/app/models/form/preview_card_provider_batch.rb new file mode 100644 index 000000000..e6ab3d8fa --- /dev/null +++ b/app/models/form/preview_card_provider_batch.rb @@ -0,0 +1,33 @@ +# frozen_string_literal: true + +class Form::PreviewCardProviderBatch + include ActiveModel::Model + include Authorization + + attr_accessor :preview_card_provider_ids, :action, :current_account + + def save + case action + when 'approve' + approve! + when 'reject' + reject! + end + end + + private + + def preview_card_providers + PreviewCardProvider.where(id: preview_card_provider_ids) + end + + def approve! + preview_card_providers.each { |provider| authorize(provider, :update?) } + preview_card_providers.update_all(trendable: true, reviewed_at: Time.now.utc) + end + + def reject! + preview_card_providers.each { |provider| authorize(provider, :update?) } + preview_card_providers.update_all(trendable: false, reviewed_at: Time.now.utc) + end +end diff --git a/app/models/form/tag_batch.rb b/app/models/form/tag_batch.rb index fd517a1a6..b9330745f 100644 --- a/app/models/form/tag_batch.rb +++ b/app/models/form/tag_batch.rb @@ -23,11 +23,15 @@ class Form::TagBatch def approve! tags.each { |tag| authorize(tag, :update?) } - tags.update_all(trendable: true, reviewed_at: Time.now.utc) + tags.update_all(trendable: true, reviewed_at: action_time) end def reject! tags.each { |tag| authorize(tag, :update?) } - tags.update_all(trendable: false, reviewed_at: Time.now.utc) + tags.update_all(trendable: false, reviewed_at: action_time) + end + + def action_time + @action_time ||= Time.now.utc end end diff --git a/app/models/preview_card.rb b/app/models/preview_card.rb index bca3a3ce8..f2ab8ecab 100644 --- a/app/models/preview_card.rb +++ b/app/models/preview_card.rb @@ -24,6 +24,11 @@ # embed_url :string default(""), not null # image_storage_schema_version :integer # blurhash :string +# language :string +# max_score :float +# max_score_at :datetime +# trendable :boolean +# link_type :integer # class PreviewCard < ApplicationRecord @@ -40,6 +45,7 @@ class PreviewCard < ApplicationRecord self.inheritance_column = false enum type: [:link, :photo, :video, :rich] + enum link_type: [:unknown, :article] has_and_belongs_to_many :statuses @@ -54,6 +60,32 @@ class PreviewCard < ApplicationRecord before_save :extract_dimensions, if: :link? + def appropriate_for_trends? + link? && article? && title.present? && description.present? && image.present? && provider_name.present? + end + + def domain + @domain ||= Addressable::URI.parse(url).normalized_host + end + + def provider + @provider ||= PreviewCardProvider.matching_domain(domain) + end + + def trendable? + if attributes['trendable'].nil? + provider&.trendable? + else + attributes['trendable'] + end + end + + def requires_review_notification? + attributes['trendable'].nil? && (provider.nil? || provider.requires_review_notification?) + end + + attr_writer :provider + def local? false end @@ -69,11 +101,14 @@ class PreviewCard < ApplicationRecord save! end + def history + @history ||= Trends::History.new('links', id) + end + class << self private - # rubocop:disable Naming/MethodParameterName - def image_styles(f) + def image_styles(file) styles = { original: { geometry: '400x400>', @@ -83,10 +118,9 @@ class PreviewCard < ApplicationRecord }, } - styles[:original][:format] = 'jpg' if f.instance.image_content_type == 'image/gif' + styles[:original][:format] = 'jpg' if file.instance.image_content_type == 'image/gif' styles end - # rubocop:enable Naming/MethodParameterName end private diff --git a/app/models/preview_card_filter.rb b/app/models/preview_card_filter.rb new file mode 100644 index 000000000..8dda9989c --- /dev/null +++ b/app/models/preview_card_filter.rb @@ -0,0 +1,53 @@ +# frozen_string_literal: true + +class PreviewCardFilter + KEYS = %i( + trending + ).freeze + + attr_reader :params + + def initialize(params) + @params = params + end + + def results + scope = PreviewCard.unscoped + + params.each do |key, value| + next if key.to_s == 'page' + + scope.merge!(scope_for(key, value.to_s.strip)) if value.present? + end + + scope + end + + private + + def scope_for(key, value) + case key.to_s + when 'trending' + trending_scope(value) + else + raise "Unknown filter: #{key}" + end + end + + def trending_scope(value) + ids = begin + case value.to_s + when 'allowed' + Trends.links.currently_trending_ids(true, -1) + else + Trends.links.currently_trending_ids(false, -1) + end + end + + if ids.empty? + PreviewCard.none + else + PreviewCard.joins("join unnest(array[#{ids.map(&:to_i).join(',')}]::integer[]) with ordinality as x (id, ordering) on preview_cards.id = x.id").order('x.ordering') + end + end +end diff --git a/app/models/preview_card_provider.rb b/app/models/preview_card_provider.rb new file mode 100644 index 000000000..15b24e2bd --- /dev/null +++ b/app/models/preview_card_provider.rb @@ -0,0 +1,57 @@ +# frozen_string_literal: true +# == Schema Information +# +# Table name: preview_card_providers +# +# id :bigint(8) not null, primary key +# domain :string default(""), not null +# icon_file_name :string +# icon_content_type :string +# icon_file_size :bigint(8) +# icon_updated_at :datetime +# trendable :boolean +# reviewed_at :datetime +# requested_review_at :datetime +# created_at :datetime not null +# updated_at :datetime not null +# + +class PreviewCardProvider < ApplicationRecord + include DomainNormalizable + include Attachmentable + + ICON_MIME_TYPES = %w(image/x-icon image/vnd.microsoft.icon image/png).freeze + LIMIT = 1.megabyte + + validates :domain, presence: true, uniqueness: true, domain: true + + has_attached_file :icon, styles: { static: { format: 'png', convert_options: '-coalesce -strip' } }, validate_media_type: false + validates_attachment :icon, content_type: { content_type: ICON_MIME_TYPES }, size: { less_than: LIMIT } + remotable_attachment :icon, LIMIT + + scope :trendable, -> { where(trendable: true) } + scope :not_trendable, -> { where(trendable: false) } + scope :reviewed, -> { where.not(reviewed_at: nil) } + scope :pending_review, -> { where(reviewed_at: nil) } + + def requires_review? + reviewed_at.nil? + end + + def reviewed? + reviewed_at.present? + end + + def requested_review? + requested_review_at.present? + end + + def requires_review_notification? + requires_review? && !requested_review? + end + + def self.matching_domain(domain) + segments = domain.split('.') + where(domain: segments.map.with_index { |_, i| segments[i..-1].join('.') }).order(Arel.sql('char_length(domain) desc')).first + end +end diff --git a/app/models/preview_card_provider_filter.rb b/app/models/preview_card_provider_filter.rb new file mode 100644 index 000000000..1e90d3c9d --- /dev/null +++ b/app/models/preview_card_provider_filter.rb @@ -0,0 +1,49 @@ +# frozen_string_literal: true + +class PreviewCardProviderFilter + KEYS = %i( + status + ).freeze + + attr_reader :params + + def initialize(params) + @params = params + end + + def results + scope = PreviewCardProvider.unscoped + + params.each do |key, value| + next if key.to_s == 'page' + + scope.merge!(scope_for(key, value.to_s.strip)) if value.present? + end + + scope.order(domain: :asc) + end + + private + + def scope_for(key, value) + case key.to_s + when 'status' + status_scope(value) + else + raise "Unknown filter: #{key}" + end + end + + def status_scope(value) + case value.to_s + when 'approved' + PreviewCardProvider.trendable + when 'rejected' + PreviewCardProvider.not_trendable + when 'pending_review' + PreviewCardProvider.pending_review + else + raise "Unknown status: #{value}" + end + end +end diff --git a/app/models/tag.rb b/app/models/tag.rb index dcce28391..f35d92b5d 100644 --- a/app/models/tag.rb +++ b/app/models/tag.rb @@ -36,6 +36,7 @@ class Tag < ApplicationRecord scope :usable, -> { where(usable: [true, nil]) } scope :listable, -> { where(listable: [true, nil]) } scope :trendable, -> { Setting.trendable_by_default ? where(trendable: [true, nil]) : where(trendable: true) } + scope :not_trendable, -> { where(trendable: false) } scope :recently_used, ->(account) { joins(:statuses).where(statuses: { id: account.statuses.select(:id).limit(1000) }).group(:id).order(Arel.sql('count(*) desc')) } scope :matches_name, ->(term) { where(arel_table[:name].lower.matches(arel_table.lower("#{sanitize_sql_like(Tag.normalize(term))}%"), nil, true)) } # Search with case-sensitive to use B-tree index @@ -75,28 +76,12 @@ class Tag < ApplicationRecord requested_review_at.present? end - def use!(account, status: nil, at_time: Time.now.utc) - TrendingTags.record_use!(self, account, status: status, at_time: at_time) - end - - def trending? - TrendingTags.trending?(self) + def requires_review_notification? + requires_review? && !requested_review? end def history - days = [] - - 7.times do |i| - day = i.days.ago.beginning_of_day.to_i - - days << { - day: day.to_s, - uses: Redis.current.get("activity:tags:#{id}:#{day}") || '0', - accounts: Redis.current.pfcount("activity:tags:#{id}:#{day}:accounts").to_s, - } - end - - days + @history ||= Trends::History.new('tags', id) end class << self diff --git a/app/models/tag_filter.rb b/app/models/tag_filter.rb index 85bfcbea5..ecdb52503 100644 --- a/app/models/tag_filter.rb +++ b/app/models/tag_filter.rb @@ -2,13 +2,8 @@ class TagFilter KEYS = %i( - directory - reviewed - unreviewed - pending_review - popular - active - name + trending + status ).freeze attr_reader :params @@ -18,7 +13,13 @@ class TagFilter end def results - scope = Tag.unscoped + scope = begin + if params[:status] == 'pending_review' + Tag.unscoped + else + trending_scope + end + end params.each do |key, value| next if key.to_s == 'page' @@ -26,27 +27,40 @@ class TagFilter scope.merge!(scope_for(key, value.to_s.strip)) if value.present? end - scope.order(id: :desc) + scope end private def scope_for(key, value) case key.to_s - when 'reviewed' - Tag.reviewed.order(reviewed_at: :desc) - when 'unreviewed' - Tag.unreviewed - when 'pending_review' - Tag.pending_review.order(requested_review_at: :desc) - when 'popular' - Tag.order('max_score DESC NULLS LAST') - when 'active' - Tag.order('last_status_at DESC NULLS LAST') - when 'name' - Tag.matches_name(value) + when 'status' + status_scope(value) else raise "Unknown filter: #{key}" end end + + def trending_scope + ids = Trends.tags.currently_trending_ids(false, -1) + + if ids.empty? + Tag.none + else + Tag.joins("join unnest(array[#{ids.map(&:to_i).join(',')}]::integer[]) with ordinality as x (id, ordering) on tags.id = x.id").order('x.ordering') + end + end + + def status_scope(value) + case value.to_s + when 'approved' + Tag.trendable + when 'rejected' + Tag.not_trendable + when 'pending_review' + Tag.pending_review + else + raise "Unknown status: #{value}" + end + end end diff --git a/app/models/trending_tags.rb b/app/models/trending_tags.rb deleted file mode 100644 index 31890b082..000000000 --- a/app/models/trending_tags.rb +++ /dev/null @@ -1,128 +0,0 @@ -# frozen_string_literal: true - -class TrendingTags - KEY = 'trending_tags' - EXPIRE_HISTORY_AFTER = 7.days.seconds - EXPIRE_TRENDS_AFTER = 1.day.seconds - THRESHOLD = 5 - LIMIT = 10 - REVIEW_THRESHOLD = 3 - MAX_SCORE_COOLDOWN = 2.days.freeze - MAX_SCORE_HALFLIFE = 2.hours.freeze - - class << self - include Redisable - - def record_use!(tag, account, status: nil, at_time: Time.now.utc) - return unless tag.usable? && !account.silenced? - - # Even if a tag is not allowed to trend, we still need to - # record the stats since they can be displayed in other places - increment_historical_use!(tag.id, at_time) - increment_unique_use!(tag.id, account.id, at_time) - increment_use!(tag.id, at_time) - - # Only update when the tag was last used once every 12 hours - # and only if a status is given (lets use ignore reblogs) - tag.update(last_status_at: at_time) if status.present? && (tag.last_status_at.nil? || (tag.last_status_at < at_time && tag.last_status_at < 12.hours.ago)) - end - - def update!(at_time = Time.now.utc) - tag_ids = redis.smembers("#{KEY}:used:#{at_time.beginning_of_day.to_i}") + redis.zrange(KEY, 0, -1) - tags = Tag.trendable.where(id: tag_ids.uniq) - - # First pass to calculate scores and update the set - - tags.each do |tag| - expected = redis.pfcount("activity:tags:#{tag.id}:#{(at_time - 1.day).beginning_of_day.to_i}:accounts").to_f - expected = 1.0 if expected.zero? - observed = redis.pfcount("activity:tags:#{tag.id}:#{at_time.beginning_of_day.to_i}:accounts").to_f - max_time = tag.max_score_at - max_score = tag.max_score - max_score = 0 if max_time.nil? || max_time < (at_time - MAX_SCORE_COOLDOWN) - - score = begin - if expected > observed || observed < THRESHOLD - 0 - else - ((observed - expected)**2) / expected - end - end - - if score > max_score - max_score = score - max_time = at_time - - # Not interested in triggering any callbacks for this - tag.update_columns(max_score: max_score, max_score_at: max_time) - end - - decaying_score = max_score * (0.5**((at_time.to_f - max_time.to_f) / MAX_SCORE_HALFLIFE.to_f)) - - if decaying_score.zero? - redis.zrem(KEY, tag.id) - else - redis.zadd(KEY, decaying_score, tag.id) - end - end - - users_for_review = User.staff.includes(:account).to_a.select(&:allows_trending_tag_emails?) - - # Second pass to notify about previously unreviewed trends - - tags.each do |tag| - current_rank = redis.zrevrank(KEY, tag.id) - needs_review_notification = tag.requires_review? && !tag.requested_review? - rank_passes_threshold = current_rank.present? && current_rank <= REVIEW_THRESHOLD - - next unless !tag.trendable? && rank_passes_threshold && needs_review_notification - - tag.touch(:requested_review_at) - - users_for_review.each do |user| - AdminMailer.new_trending_tag(user.account, tag).deliver_later! - end - end - - # Trim older items - - redis.zremrangebyrank(KEY, 0, -(LIMIT + 1)) - redis.zremrangebyscore(KEY, '(0.3', '-inf') - end - - def get(limit, filtered: true) - tag_ids = redis.zrevrange(KEY, 0, LIMIT - 1).map(&:to_i) - - tags = Tag.where(id: tag_ids) - tags = tags.trendable if filtered - tags = tags.index_by(&:id) - - tag_ids.map { |tag_id| tags[tag_id] }.compact.take(limit) - end - - def trending?(tag) - rank = redis.zrevrank(KEY, tag.id) - rank.present? && rank < LIMIT - end - - private - - def increment_historical_use!(tag_id, at_time) - key = "activity:tags:#{tag_id}:#{at_time.beginning_of_day.to_i}" - redis.incrby(key, 1) - redis.expire(key, EXPIRE_HISTORY_AFTER) - end - - def increment_unique_use!(tag_id, account_id, at_time) - key = "activity:tags:#{tag_id}:#{at_time.beginning_of_day.to_i}:accounts" - redis.pfadd(key, account_id) - redis.expire(key, EXPIRE_HISTORY_AFTER) - end - - def increment_use!(tag_id, at_time) - key = "#{KEY}:used:#{at_time.beginning_of_day.to_i}" - redis.sadd(key, tag_id) - redis.expire(key, EXPIRE_HISTORY_AFTER) - end - end -end diff --git a/app/models/trends.rb b/app/models/trends.rb new file mode 100644 index 000000000..7dd3a9c87 --- /dev/null +++ b/app/models/trends.rb @@ -0,0 +1,27 @@ +# frozen_string_literal: true + +module Trends + def self.table_name_prefix + 'trends_' + end + + def self.links + @links ||= Trends::Links.new + end + + def self.tags + @tags ||= Trends::Tags.new + end + + def self.refresh! + [links, tags].each(&:refresh) + end + + def self.request_review! + [links, tags].each(&:request_review) if enabled? + end + + def self.enabled? + Setting.trends + end +end diff --git a/app/models/trends/base.rb b/app/models/trends/base.rb new file mode 100644 index 000000000..b767dcb1a --- /dev/null +++ b/app/models/trends/base.rb @@ -0,0 +1,80 @@ +# frozen_string_literal: true + +class Trends::Base + include Redisable + + class_attribute :default_options + + attr_reader :options + + # @param [Hash] options + # @option options [Integer] :threshold Minimum amount of uses by unique accounts to begin calculating the score + # @option options [Integer] :review_threshold Minimum rank (lower = better) before requesting a review + # @option options [ActiveSupport::Duration] :max_score_cooldown For this amount of time, the peak score (if bigger than current score) is decayed-from + # @option options [ActiveSupport::Duration] :max_score_halflife How quickly a peak score decays + def initialize(options = {}) + @options = self.class.default_options.merge(options) + end + + def register(_status) + raise NotImplementedError + end + + def add(*) + raise NotImplementedError + end + + def refresh(*) + raise NotImplementedError + end + + def request_review + raise NotImplementedError + end + + def get(*) + raise NotImplementedError + end + + def score(id) + redis.zscore("#{key_prefix}:all", id) || 0 + end + + def rank(id) + redis.zrevrank("#{key_prefix}:allowed", id) + end + + def currently_trending_ids(allowed, limit) + redis.zrevrange(allowed ? "#{key_prefix}:allowed" : "#{key_prefix}:all", 0, limit.positive? ? limit - 1 : limit).map(&:to_i) + end + + protected + + def key_prefix + raise NotImplementedError + end + + def recently_used_ids(at_time = Time.now.utc) + redis.smembers(used_key(at_time)).map(&:to_i) + end + + def record_used_id(id, at_time = Time.now.utc) + redis.sadd(used_key(at_time), id) + redis.expire(used_key(at_time), 1.day.seconds) + end + + def trim_older_items + redis.zremrangebyscore("#{key_prefix}:all", '-inf', '(1') + redis.zremrangebyscore("#{key_prefix}:allowed", '-inf', '(1') + end + + def score_at_rank(rank) + redis.zrevrange("#{key_prefix}:allowed", 0, rank, with_scores: true).last&.last || 0 + end + + private + + def used_key(at_time) + "#{key_prefix}:used:#{at_time.beginning_of_day.to_i}" + end +end diff --git a/app/models/trends/history.rb b/app/models/trends/history.rb new file mode 100644 index 000000000..608e33792 --- /dev/null +++ b/app/models/trends/history.rb @@ -0,0 +1,98 @@ +# frozen_string_literal: true + +class Trends::History + include Enumerable + + class Aggregate + include Redisable + + def initialize(prefix, id, date_range) + @days = date_range.map { |date| Day.new(prefix, id, date.to_time(:utc)) } + end + + def uses + redis.mget(*@days.map { |day| day.key_for(:uses) }).map(&:to_i).sum + end + + def accounts + redis.pfcount(*@days.map { |day| day.key_for(:accounts) }) + end + end + + class Day + include Redisable + + EXPIRE_AFTER = 14.days.seconds + + def initialize(prefix, id, day) + @prefix = prefix + @id = id + @day = day.beginning_of_day + end + + attr_reader :day + + def accounts + redis.pfcount(key_for(:accounts)) + end + + def uses + redis.get(key_for(:uses))&.to_i || 0 + end + + def add(account_id) + redis.pipelined do + redis.incrby(key_for(:uses), 1) + redis.pfadd(key_for(:accounts), account_id) + redis.expire(key_for(:uses), EXPIRE_AFTER) + redis.expire(key_for(:accounts), EXPIRE_AFTER) + end + end + + def as_json + { day: day.to_i.to_s, accounts: accounts.to_s, uses: uses.to_s } + end + + def key_for(suffix) + case suffix + when :accounts + "#{key_prefix}:#{suffix}" + when :uses + key_prefix + end + end + + def key_prefix + "activity:#{@prefix}:#{@id}:#{day.to_i}" + end + end + + def initialize(prefix, id) + @prefix = prefix + @id = id + end + + def get(date) + Day.new(@prefix, @id, date) + end + + def add(account_id, at_time = Time.now.utc) + Day.new(@prefix, @id, at_time).add(account_id) + end + + def aggregate(date_range) + Aggregate.new(@prefix, @id, date_range) + end + + def each(&block) + if block_given? + (0...7).map { |i| block.call(get(i.days.ago)) } + else + to_enum(:each) + end + end + + def as_json(*) + map(&:as_json) + end +end diff --git a/app/models/trends/links.rb b/app/models/trends/links.rb new file mode 100644 index 000000000..a0d65138b --- /dev/null +++ b/app/models/trends/links.rb @@ -0,0 +1,117 @@ +# frozen_string_literal: true + +class Trends::Links < Trends::Base + PREFIX = 'trending_links' + + self.default_options = { + threshold: 15, + review_threshold: 10, + max_score_cooldown: 2.days.freeze, + max_score_halflife: 8.hours.freeze, + } + + def register(status, at_time = Time.now.utc) + original_status = status.reblog? ? status.reblog : status + + return unless original_status.public_visibility? && status.public_visibility? && + !original_status.account.silenced? && !status.account.silenced? && + !original_status.spoiler_text? + + original_status.preview_cards.each do |preview_card| + add(preview_card, status.account_id, at_time) if preview_card.appropriate_for_trends? + end + end + + def add(preview_card, account_id, at_time = Time.now.utc) + preview_card.history.add(account_id, at_time) + record_used_id(preview_card.id, at_time) + end + + def get(allowed, limit) + preview_card_ids = currently_trending_ids(allowed, limit) + preview_cards = PreviewCard.where(id: preview_card_ids).index_by(&:id) + preview_card_ids.map { |id| preview_cards[id] }.compact + end + + def refresh(at_time = Time.now.utc) + preview_cards = PreviewCard.where(id: (recently_used_ids(at_time) + currently_trending_ids(false, -1)).uniq) + calculate_scores(preview_cards, at_time) + trim_older_items + end + + def request_review + preview_cards = PreviewCard.where(id: currently_trending_ids(false, -1)) + + preview_cards_requiring_review = preview_cards.filter_map do |preview_card| + next unless would_be_trending?(preview_card.id) && !preview_card.trendable? && preview_card.requires_review_notification? + + if preview_card.provider.nil? + preview_card.provider = PreviewCardProvider.create(domain: preview_card.domain, requested_review_at: Time.now.utc) + else + preview_card.provider.touch(:requested_review_at) + end + + preview_card + end + + return if preview_cards_requiring_review.empty? + + User.staff.includes(:account).find_each do |user| + AdminMailer.new_trending_links(user.account, preview_cards_requiring_review).deliver_later! if user.allows_trending_tag_emails? + end + end + + protected + + def key_prefix + PREFIX + end + + private + + def calculate_scores(preview_cards, at_time) + preview_cards.each do |preview_card| + expected = preview_card.history.get(at_time - 1.day).accounts.to_f + expected = 1.0 if expected.zero? + observed = preview_card.history.get(at_time).accounts.to_f + max_time = preview_card.max_score_at + max_score = preview_card.max_score + max_score = 0 if max_time.nil? || max_time < (at_time - options[:max_score_cooldown]) + + score = begin + if expected > observed || observed < options[:threshold] + 0 + else + ((observed - expected)**2) / expected + end + end + + if score > max_score + max_score = score + max_time = at_time + + # Not interested in triggering any callbacks for this + preview_card.update_columns(max_score: max_score, max_score_at: max_time) + end + + decaying_score = max_score * (0.5**((at_time.to_f - max_time.to_f) / options[:max_score_halflife].to_f)) + + if decaying_score.zero? + redis.zrem("#{PREFIX}:all", preview_card.id) + redis.zrem("#{PREFIX}:allowed", preview_card.id) + else + redis.zadd("#{PREFIX}:all", decaying_score, preview_card.id) + + if preview_card.trendable? + redis.zadd("#{PREFIX}:allowed", decaying_score, preview_card.id) + else + redis.zrem("#{PREFIX}:allowed", preview_card.id) + end + end + end + end + + def would_be_trending?(id) + score(id) > score_at_rank(options[:review_threshold] - 1) + end +end diff --git a/app/models/trends/tags.rb b/app/models/trends/tags.rb new file mode 100644 index 000000000..13e0ab56b --- /dev/null +++ b/app/models/trends/tags.rb @@ -0,0 +1,111 @@ +# frozen_string_literal: true + +class Trends::Tags < Trends::Base + PREFIX = 'trending_tags' + + self.default_options = { + threshold: 15, + review_threshold: 10, + max_score_cooldown: 2.days.freeze, + max_score_halflife: 4.hours.freeze, + } + + def register(status, at_time = Time.now.utc) + original_status = status.reblog? ? status.reblog : status + + return unless original_status.public_visibility? && status.public_visibility? && + !original_status.account.silenced? && !status.account.silenced? + + original_status.tags.each do |tag| + add(tag, status.account_id, at_time) if tag.usable? + end + end + + def add(tag, account_id, at_time = Time.now.utc) + tag.history.add(account_id, at_time) + record_used_id(tag.id, at_time) + end + + def refresh(at_time = Time.now.utc) + tags = Tag.where(id: (recently_used_ids(at_time) + currently_trending_ids(false, -1)).uniq) + calculate_scores(tags, at_time) + trim_older_items + end + + def get(allowed, limit) + tag_ids = currently_trending_ids(allowed, limit) + tags = Tag.where(id: tag_ids).index_by(&:id) + tag_ids.map { |id| tags[id] }.compact + end + + def request_review + tags = Tag.where(id: currently_trending_ids(false, -1)) + + tags_requiring_review = tags.filter_map do |tag| + next unless would_be_trending?(tag.id) && !tag.trendable? && tag.requires_review_notification? + + tag.touch(:requested_review_at) + tag + end + + return if tags_requiring_review.empty? + + User.staff.includes(:account).find_each do |user| + AdminMailer.new_trending_tags(user.account, tags_requiring_review).deliver_later! if user.allows_trending_tag_emails? + end + end + + protected + + def key_prefix + PREFIX + end + + private + + def calculate_scores(tags, at_time) + tags.each do |tag| + expected = tag.history.get(at_time - 1.day).accounts.to_f + expected = 1.0 if expected.zero? + observed = tag.history.get(at_time).accounts.to_f + max_time = tag.max_score_at + max_score = tag.max_score + max_score = 0 if max_time.nil? || max_time < (at_time - options[:max_score_cooldown]) + + score = begin + if expected > observed || observed < options[:threshold] + 0 + else + ((observed - expected)**2) / expected + end + end + + if score > max_score + max_score = score + max_time = at_time + + # Not interested in triggering any callbacks for this + tag.update_columns(max_score: max_score, max_score_at: max_time) + end + + decaying_score = max_score * (0.5**((at_time.to_f - max_time.to_f) / options[:max_score_halflife].to_f)) + + if decaying_score.zero? + redis.zrem("#{PREFIX}:all", tag.id) + redis.zrem("#{PREFIX}:allowed", tag.id) + else + redis.zadd("#{PREFIX}:all", decaying_score, tag.id) + + if tag.trendable? + redis.zadd("#{PREFIX}:allowed", decaying_score, tag.id) + else + redis.zrem("#{PREFIX}:allowed", tag.id) + end + end + end + end + + def would_be_trending?(id) + score(id) > score_at_rank(options[:review_threshold] - 1) + end +end -- cgit