diff options
author | Eugen Rochko <eugen@zeonfederated.com> | 2022-04-29 22:42:42 +0200 |
---|---|---|
committer | GitHub <noreply@github.com> | 2022-04-29 22:42:42 +0200 |
commit | 6476f7e4da4da7c353d497aae5a86fc3909ce532 (patch) | |
tree | c800ed4deac373a82298d962679c1bb066d261f0 /app/models/trends/statuses.rb | |
parent | 5c691430e5f05524523bf32a3fce10633ec37092 (diff) |
Change trending statuses to only show one status from each account (#18181)
Calculate trends in temporary sets to avoid having to manage items that go below the decay threshold while not having any moments where a half-processed set is accessible to end-users
Diffstat (limited to 'app/models/trends/statuses.rb')
-rw-r--r-- | app/models/trends/statuses.rb | 67 |
1 files changed, 30 insertions, 37 deletions
diff --git a/app/models/trends/statuses.rb b/app/models/trends/statuses.rb index 3013bc1d1..777065d3e 100644 --- a/app/models/trends/statuses.rb +++ b/app/models/trends/statuses.rb @@ -7,6 +7,7 @@ class Trends::Statuses < Trends::Base threshold: 5, review_threshold: 3, score_halflife: 2.hours.freeze, + decay_threshold: 0.3, } class Query < Trends::Query @@ -31,7 +32,7 @@ class Trends::Statuses < Trends::Base end def register(status, at_time = Time.now.utc) - add(status.proper, status.account_id, at_time) if eligible?(status) + add(status.proper, status.account_id, at_time) if eligible?(status.proper) end def add(status, _account_id, at_time = Time.now.utc) @@ -74,53 +75,45 @@ class Trends::Statuses < Trends::Base private def eligible?(status) - original_status = status.proper - - original_status.public_visibility? && - original_status.account.discoverable? && !original_status.account.silenced? && - original_status.spoiler_text.blank? && !original_status.sensitive? && !original_status.reply? + status.public_visibility? && status.account.discoverable? && !status.account.silenced? && status.spoiler_text.blank? && !status.sensitive? && !status.reply? end def calculate_scores(statuses, at_time) - redis.pipelined do - statuses.each do |status| - expected = 1.0 - observed = (status.reblogs_count + status.favourites_count).to_f - - score = begin - if expected > observed || observed < options[:threshold] - 0 - else - ((observed - expected)**2) / expected - end + global_items = [] + locale_items = Hash.new { |h, key| h[key] = [] } + + statuses.each do |status| + expected = 1.0 + observed = (status.reblogs_count + status.favourites_count).to_f + + score = begin + if expected > observed || observed < options[:threshold] + 0 + else + ((observed - expected)**2) / expected end + end - decaying_score = score * (0.5**((at_time.to_f - status.created_at.to_f) / options[:score_halflife].to_f)) + decaying_score = score * (0.5**((at_time.to_f - status.created_at.to_f) / options[:score_halflife].to_f)) - add_to_and_remove_from_subsets(status.id, decaying_score, { - all: true, - allowed: status.trendable? && status.account.discoverable?, - }) + next unless decaying_score >= options[:decay_threshold] - next unless valid_locale?(status.language) + global_items << { score: decaying_score, item: status } + locale_items[status.language] << { account_id: status.account_id, score: decaying_score, item: status } if valid_locale?(status.language) + end - add_to_and_remove_from_subsets(status.id, decaying_score, { - "all:#{status.language}" => true, - "allowed:#{status.language}" => status.trendable? && status.account.discoverable?, - }) - end + replace_items('', global_items) - trim_older_items + Trends.available_locales.each do |locale| + replace_items(":#{locale}", locale_items[locale]) + end + end - # Clean up localized sets by calculating the intersection with the main - # set. We do this instead of just deleting the localized sets to avoid - # having moments where the API returns empty results + def filter_for_allowed_items(items) + # Show only one status per account, pick the one with the highest score + # that's also eligible to trend - Trends.available_locales.each do |locale| - redis.zinterstore("#{key_prefix}:all:#{locale}", ["#{key_prefix}:all:#{locale}", "#{key_prefix}:all"], aggregate: 'max') - redis.zinterstore("#{key_prefix}:allowed:#{locale}", ["#{key_prefix}:allowed:#{locale}", "#{key_prefix}:allowed"], aggregate: 'max') - end - end + items.group_by { |item| item[:account_id] }.values.filter_map { |account_items| account_items.select { |item| item[:item].trendable? && item[:item].account.discoverable? }.max_by { |item| item[:score] } } end def would_be_trending?(id) |