From 97192d9a77c0b4b68afe50d6a94d87110a8adbcd Mon Sep 17 00:00:00 2001 From: Eugen Rochko Date: Thu, 22 Aug 2019 04:17:12 +0200 Subject: Fix remote and staff-removed statuses leaving media behind for a day (#11638) The reason for unattaching media instead of removing it is to support delete & redraft functionality, but remote or staff-removed statuses will never be redrafted, so the media should be deleted immediately --- spec/models/form/status_batch_spec.rb | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) (limited to 'spec/models') diff --git a/spec/models/form/status_batch_spec.rb b/spec/models/form/status_batch_spec.rb index 00c790a11..f9c58c90f 100644 --- a/spec/models/form/status_batch_spec.rb +++ b/spec/models/form/status_batch_spec.rb @@ -41,12 +41,12 @@ describe Form::StatusBatch do it 'call RemovalWorker' do form.save - expect(RemovalWorker).to have_received(:perform_async).with(status.id) + expect(RemovalWorker).to have_received(:perform_async).with(status.id, redraft: false) end it 'do not call RemovalWorker' do form.save - expect(RemovalWorker).not_to have_received(:perform_async).with(another_status.id) + expect(RemovalWorker).not_to have_received(:perform_async).with(another_status.id, redraft: false) end end end -- cgit From 73ca0bb925cb036f824262ab292a157a40a515d0 Mon Sep 17 00:00:00 2001 From: Eugen Rochko Date: Fri, 23 Aug 2019 22:37:23 +0200 Subject: Add option to include reported statuses in warning e-mail (#11639) --- .../admin/account_actions_controller.rb | 4 ++-- app/javascript/styles/mailer.scss | 7 ++++++ app/mailers/user_mailer.rb | 4 +++- app/models/admin/account_action.rb | 22 +++++++++++++----- app/views/admin/account_actions/new.html.haml | 4 ++++ app/views/notification_mailer/_status.html.haml | 8 ++++++- app/views/user_mailer/warning.html.haml | 27 +++++++++++++++++++++- app/views/user_mailer/warning.text.erb | 13 +++++++++++ config/locales/en.yml | 2 ++ config/locales/simple_form.en.yml | 2 ++ spec/mailers/previews/user_mailer_preview.rb | 2 +- spec/models/admin/account_action_spec.rb | 4 ++-- 12 files changed, 85 insertions(+), 14 deletions(-) (limited to 'spec/models') diff --git a/app/controllers/admin/account_actions_controller.rb b/app/controllers/admin/account_actions_controller.rb index a2cea461e..ea56fa0ac 100644 --- a/app/controllers/admin/account_actions_controller.rb +++ b/app/controllers/admin/account_actions_controller.rb @@ -5,7 +5,7 @@ module Admin before_action :set_account def new - @account_action = Admin::AccountAction.new(type: params[:type], report_id: params[:report_id], send_email_notification: true) + @account_action = Admin::AccountAction.new(type: params[:type], report_id: params[:report_id], send_email_notification: true, include_statuses: true) @warning_presets = AccountWarningPreset.all end @@ -30,7 +30,7 @@ module Admin end def resource_params - params.require(:admin_account_action).permit(:type, :report_id, :warning_preset_id, :text, :send_email_notification) + params.require(:admin_account_action).permit(:type, :report_id, :warning_preset_id, :text, :send_email_notification, :include_statuses) end end end diff --git a/app/javascript/styles/mailer.scss b/app/javascript/styles/mailer.scss index b4fb1d709..e25a80c04 100644 --- a/app/javascript/styles/mailer.scss +++ b/app/javascript/styles/mailer.scss @@ -457,6 +457,13 @@ h5 { .status { padding-bottom: 32px; + &--highlighted { + border: 1px solid lighten($ui-base-color, 8%); + border-radius: 4px; + padding-bottom: 16px; + margin-bottom: 16px; + } + .status-header { td { font-size: 14px; diff --git a/app/mailers/user_mailer.rb b/app/mailers/user_mailer.rb index 8f3a4ab3a..b41004acc 100644 --- a/app/mailers/user_mailer.rb +++ b/app/mailers/user_mailer.rb @@ -5,6 +5,7 @@ class UserMailer < Devise::Mailer helper :application helper :instance + helper :statuses add_template_helper RoutingHelper @@ -79,10 +80,11 @@ class UserMailer < Devise::Mailer end end - def warning(user, warning) + def warning(user, warning, status_ids = nil) @resource = user @warning = warning @instance = Rails.configuration.x.local_domain + @statuses = Status.where(id: status_ids).includes(:account) if status_ids.is_a?(Array) I18n.with_locale(@resource.locale || I18n.default_locale) do mail to: @resource.email, diff --git a/app/models/admin/account_action.rb b/app/models/admin/account_action.rb index bdbd342fb..c7da8b52c 100644 --- a/app/models/admin/account_action.rb +++ b/app/models/admin/account_action.rb @@ -19,20 +19,25 @@ class Admin::AccountAction :report_id, :warning_preset_id - attr_reader :warning, :send_email_notification + attr_reader :warning, :send_email_notification, :include_statuses def send_email_notification=(value) @send_email_notification = ActiveModel::Type::Boolean.new.cast(value) end + def include_statuses=(value) + @include_statuses = ActiveModel::Type::Boolean.new.cast(value) + end + def save! ApplicationRecord.transaction do process_action! process_warning! end - queue_email! + process_email! process_reports! + process_queue! end def report @@ -110,7 +115,6 @@ class Admin::AccountAction authorize(target_account, :suspend?) log_action(:suspend, target_account) target_account.suspend! - queue_suspension_worker! end def text_for_warning @@ -121,16 +125,22 @@ class Admin::AccountAction Admin::SuspensionWorker.perform_async(target_account.id) end - def queue_email! - return unless warnable? + def process_queue! + queue_suspension_worker! if type == 'suspend' + end - UserMailer.warning(target_account.user, warning).deliver_later! + def process_email! + UserMailer.warning(target_account.user, warning, status_ids).deliver_now! if warnable? end def warnable? send_email_notification && target_account.local? end + def status_ids + @report.status_ids if @report && include_statuses + end + def warning_preset @warning_preset ||= AccountWarningPreset.find(warning_preset_id) if warning_preset_id.present? end diff --git a/app/views/admin/account_actions/new.html.haml b/app/views/admin/account_actions/new.html.haml index 97286c8e5..20fbeef33 100644 --- a/app/views/admin/account_actions/new.html.haml +++ b/app/views/admin/account_actions/new.html.haml @@ -13,6 +13,10 @@ .fields-group = f.input :send_email_notification, as: :boolean, wrapper: :with_label + - if params[:report_id].present? + .fields-group + = f.input :include_statuses, as: :boolean, wrapper: :with_label + %hr.spacer/ - unless @warning_presets.empty? diff --git a/app/views/notification_mailer/_status.html.haml b/app/views/notification_mailer/_status.html.haml index 57b5688bd..40f3aa88a 100644 --- a/app/views/notification_mailer/_status.html.haml +++ b/app/views/notification_mailer/_status.html.haml @@ -1,4 +1,5 @@ - i ||= 0 +- highlighted ||= false %table.email-table{ cellspacing: 0, cellpadding: 0, dir: 'ltr' } %tbody @@ -14,7 +15,7 @@ %table.column{ cellspacing: 0, cellpadding: 0 } %tbody %tr - %td.column-cell.padded.status + %td.column-cell.padded.status{ class: highlighted ? 'status--highlighted' : '' } %table.status-header{ cellspacing: 0, cellpadding: 0 } %tbody %tr @@ -32,5 +33,10 @@ %div{ dir: rtl_status?(status) ? 'rtl' : 'ltr' } = Formatter.instance.format(status) + - if status.media_attachments.size > 0 + %p + - status.media_attachments.each do |a| + = link_to medium_url(a), medium_url(a) + %p.status-footer = link_to l(status.created_at), web_url("statuses/#{status.id}") diff --git a/app/views/user_mailer/warning.html.haml b/app/views/user_mailer/warning.html.haml index 72ea5e5d2..030a57bb4 100644 --- a/app/views/user_mailer/warning.html.haml +++ b/app/views/user_mailer/warning.html.haml @@ -42,6 +42,14 @@ - unless @warning.text.blank? = Formatter.instance.linkify(@warning.text) + - unless @statuses.empty? + %p + %strong= t('user_mailer.warning.statuses') + +- unless @statuses.empty? + - @statuses.each_with_index do |status, i| + = render 'notification_mailer/status', status: status, i: i + 1, highlighted: true + %table.email-table{ cellspacing: 0, cellpadding: 0 } %tbody %tr @@ -50,7 +58,7 @@ %table.content-section{ cellspacing: 0, cellpadding: 0 } %tbody %tr - %td.content-cell + %td.content-cell{ class: @statuses.empty? ? '' : 'content-start' } %table.column{ cellspacing: 0, cellpadding: 0 } %tbody %tr @@ -61,3 +69,20 @@ %td.button-primary = link_to about_more_url do %span= t 'user_mailer.warning.review_server_policies' + +%table.email-table{ cellspacing: 0, cellpadding: 0 } + %tbody + %tr + %td.email-body + .email-container + %table.content-section{ cellspacing: 0, cellpadding: 0 } + %tbody + %tr + %td.content-cell + .email-row + .col-6 + %table.column{ cellspacing: 0, cellpadding: 0 } + %tbody + %tr + %td.column-cell.text-center + %p= t 'user_mailer.warning.get_in_touch', instance: @instance diff --git a/app/views/user_mailer/warning.text.erb b/app/views/user_mailer/warning.text.erb index b4f2402cb..24c1f86f2 100644 --- a/app/views/user_mailer/warning.text.erb +++ b/app/views/user_mailer/warning.text.erb @@ -7,3 +7,16 @@ <% end %> <%= @warning.text %> +<% unless @statuses.empty? %> +<%= t('user_mailer.warning.statuses') %> + +<% @statuses.each do |status| %> + +<%= render 'notification_mailer/status', status: status %> +--- +<% end %> +<% else %> +--- +<% end %> + +<%= t 'user_mailer.warning.get_in_touch', instance: @instance %> diff --git a/config/locales/en.yml b/config/locales/en.yml index a50dcb8a5..ee78e4720 100644 --- a/config/locales/en.yml +++ b/config/locales/en.yml @@ -1115,7 +1115,9 @@ en: disable: While your account is frozen, your account data remains intact, but you cannot perform any actions until it is unlocked. silence: While your account is limited, only people who are already following you will see your toots on this server, and you may be excluded from various public listings. However, others may still manually follow you. suspend: Your account has been suspended, and all of your toots and your uploaded media files have been irreversibly removed from this server, and servers where you had followers. + get_in_touch: You can reply to this e-mail to get in touch with the staff of %{instance}. review_server_policies: Review server policies + statuses: 'Specifically, for:' subject: disable: Your account %{acct} has been frozen none: Warning for %{acct} diff --git a/config/locales/simple_form.en.yml b/config/locales/simple_form.en.yml index 98f0843d0..cfaa6e666 100644 --- a/config/locales/simple_form.en.yml +++ b/config/locales/simple_form.en.yml @@ -5,6 +5,7 @@ en: account_warning_preset: text: You can use toot syntax, such as URLs, hashtags and mentions admin_account_action: + include_statuses: The user will see which toots have caused the moderation action or warning send_email_notification: The user will receive an explanation of what happened with their account text_html: Optional. You can use toot syntax. You can add warning presets to save time type_html: Choose what to do with %{acct} @@ -60,6 +61,7 @@ en: account_warning_preset: text: Preset text admin_account_action: + include_statuses: Include reported toots in the e-mail send_email_notification: Notify the user per e-mail text: Custom warning type: Action diff --git a/spec/mailers/previews/user_mailer_preview.rb b/spec/mailers/previews/user_mailer_preview.rb index 53c836494..ead3b3baa 100644 --- a/spec/mailers/previews/user_mailer_preview.rb +++ b/spec/mailers/previews/user_mailer_preview.rb @@ -42,6 +42,6 @@ class UserMailerPreview < ActionMailer::Preview # Preview this email at http://localhost:3000/rails/mailers/user_mailer/warning def warning - UserMailer.warning(User.first, AccountWarning.new(text: '', action: :silence)) + UserMailer.warning(User.first, AccountWarning.new(text: '', action: :silence), [Status.first.id]) end end diff --git a/spec/models/admin/account_action_spec.rb b/spec/models/admin/account_action_spec.rb index a3db60cfc..87fc28500 100644 --- a/spec/models/admin/account_action_spec.rb +++ b/spec/models/admin/account_action_spec.rb @@ -58,8 +58,8 @@ RSpec.describe Admin::AccountAction, type: :model do end.to change { Admin::ActionLog.count }.by 1 end - it 'calls queue_email!' do - expect(account_action).to receive(:queue_email!) + it 'calls process_email!' do + expect(account_action).to receive(:process_email!) subject end -- cgit From b54b725d6bc8dd0a4ab0fe0bf408193c1bae8106 Mon Sep 17 00:00:00 2001 From: Eugen Rochko Date: Fri, 30 Aug 2019 02:19:17 +0200 Subject: Fix uncaught domain normalization error in remote follow (#11703) --- app/controllers/remote_follow_controller.rb | 2 +- app/controllers/remote_interaction_controller.rb | 2 +- app/models/remote_follow.rb | 6 ++++-- app/validators/domain_validator.rb | 12 ++++++++++-- spec/controllers/remote_follow_controller_spec.rb | 4 +--- spec/models/remote_follow_spec.rb | 2 +- 6 files changed, 18 insertions(+), 10 deletions(-) (limited to 'spec/models') diff --git a/app/controllers/remote_follow_controller.rb b/app/controllers/remote_follow_controller.rb index 0fb71d335..ba963a7a0 100644 --- a/app/controllers/remote_follow_controller.rb +++ b/app/controllers/remote_follow_controller.rb @@ -29,7 +29,7 @@ class RemoteFollowController < ApplicationController end def session_params - { acct: session[:remote_follow] } + { acct: session[:remote_follow] || current_account&.username } end def set_body_classes diff --git a/app/controllers/remote_interaction_controller.rb b/app/controllers/remote_interaction_controller.rb index de5616e25..15224e853 100644 --- a/app/controllers/remote_interaction_controller.rb +++ b/app/controllers/remote_interaction_controller.rb @@ -32,7 +32,7 @@ class RemoteInteractionController < ApplicationController end def session_params - { acct: session[:remote_follow] } + { acct: session[:remote_follow] || current_account&.username } end def set_status diff --git a/app/models/remote_follow.rb b/app/models/remote_follow.rb index 93df11724..52dd3f67b 100644 --- a/app/models/remote_follow.rb +++ b/app/models/remote_follow.rb @@ -6,7 +6,7 @@ class RemoteFollow attr_accessor :acct, :addressable_template - validates :acct, presence: true + validates :acct, presence: true, domain: { acct: true } def initialize(attrs = {}) @acct = normalize_acct(attrs[:acct]) @@ -21,7 +21,7 @@ class RemoteFollow end def subscribe_address_for(account) - addressable_template.expand(uri: account.local_username_and_domain).to_s + addressable_template.expand(uri: ActivityPub::TagManager.instance.uri_for(account)).to_s end def interact_address_for(status) @@ -44,6 +44,8 @@ class RemoteFollow end [username, domain].compact.join('@') + rescue Addressable::URI::InvalidURIError + value end def fetch_template! diff --git a/app/validators/domain_validator.rb b/app/validators/domain_validator.rb index ae07f1798..6e4a854ff 100644 --- a/app/validators/domain_validator.rb +++ b/app/validators/domain_validator.rb @@ -4,14 +4,22 @@ class DomainValidator < ActiveModel::EachValidator def validate_each(record, attribute, value) return if value.blank? - record.errors.add(attribute, I18n.t('domain_validator.invalid_domain')) unless compliant?(value) + domain = begin + if options[:acct] + value.split('@').last + else + value + end + end + + record.errors.add(attribute, I18n.t('domain_validator.invalid_domain')) unless compliant?(domain) end private def compliant?(value) Addressable::URI.new.tap { |uri| uri.host = value } - rescue Addressable::URI::InvalidURIError + rescue Addressable::URI::InvalidURIError, IDN::Idna::IdnaError false end end diff --git a/spec/controllers/remote_follow_controller_spec.rb b/spec/controllers/remote_follow_controller_spec.rb index 5088c2e65..d79dd2949 100644 --- a/spec/controllers/remote_follow_controller_spec.rb +++ b/spec/controllers/remote_follow_controller_spec.rb @@ -66,9 +66,7 @@ describe RemoteFollowController do end it 'redirects to the remote location' do - address = "http://example.com/follow_me?acct=test_user%40#{Rails.configuration.x.local_domain}" - - expect(response).to redirect_to(address) + expect(response).to redirect_to("http://example.com/follow_me?acct=https%3A%2F%2F#{Rails.configuration.x.local_domain}%2Fusers%2Ftest_user") end end end diff --git a/spec/models/remote_follow_spec.rb b/spec/models/remote_follow_spec.rb index ed2667b28..5b4c19b5b 100644 --- a/spec/models/remote_follow_spec.rb +++ b/spec/models/remote_follow_spec.rb @@ -61,7 +61,7 @@ RSpec.describe RemoteFollow do subject { remote_follow.subscribe_address_for(account) } it 'returns subscribe address' do - is_expected.to eq 'https://quitter.no/main/ostatussub?profile=alice%40cb6e6126.ngrok.io' + is_expected.to eq 'https://quitter.no/main/ostatussub?profile=https%3A%2F%2Fcb6e6126.ngrok.io%2Fusers%2Falice' end end end -- cgit From 70ddef2654a931827ce5e4323e3042365f6078f2 Mon Sep 17 00:00:00 2001 From: Eugen Rochko Date: Mon, 2 Sep 2019 18:11:13 +0200 Subject: Change trending hashtags to not disappear instantly after midnight (#11712) --- app/controllers/admin/tags_controller.rb | 2 +- app/lib/feed_manager.rb | 2 +- app/models/tag.rb | 4 +- app/models/trending_tags.rb | 102 +++++++++++++++------ app/workers/scheduler/trending_tags_scheduler.rb | 11 +++ config/sidekiq.yml | 3 + db/migrate/20190901035623_add_max_score_to_tags.rb | 6 ++ .../20190901040524_remove_score_from_tags.rb | 12 +++ db/schema.rb | 6 +- spec/models/trending_tags_spec.rb | 68 ++++++++++++++ 10 files changed, 179 insertions(+), 37 deletions(-) create mode 100644 app/workers/scheduler/trending_tags_scheduler.rb create mode 100644 db/migrate/20190901035623_add_max_score_to_tags.rb create mode 100644 db/post_migrate/20190901040524_remove_score_from_tags.rb create mode 100644 spec/models/trending_tags_spec.rb (limited to 'spec/models') diff --git a/app/controllers/admin/tags_controller.rb b/app/controllers/admin/tags_controller.rb index 25d9b7d3d..8bd4e5f8b 100644 --- a/app/controllers/admin/tags_controller.rb +++ b/app/controllers/admin/tags_controller.rb @@ -57,7 +57,7 @@ module Admin scope = scope.unreviewed if filter_params[:review] == 'unreviewed' scope = scope.reviewed.order(reviewed_at: :desc) if filter_params[:review] == 'reviewed' scope = scope.pending_review.order(requested_review_at: :desc) if filter_params[:review] == 'pending_review' - scope.order(score: :desc) + scope.order(max_score: :desc) end def filter_params diff --git a/app/lib/feed_manager.rb b/app/lib/feed_manager.rb index ca3d890a8..871ec5c19 100644 --- a/app/lib/feed_manager.rb +++ b/app/lib/feed_manager.rb @@ -63,7 +63,7 @@ class FeedManager reblog_key = key(type, account_id, 'reblogs') # Remove any items past the MAX_ITEMS'th entry in our feed - redis.zremrangebyrank(timeline_key, '0', (-(FeedManager::MAX_ITEMS + 1)).to_s) + redis.zremrangebyrank(timeline_key, 0, -(FeedManager::MAX_ITEMS + 1)) # Get the score of the REBLOG_FALLOFF'th item in our feed, and stop # tracking anything after it for deduplication purposes. diff --git a/app/models/tag.rb b/app/models/tag.rb index 945e3a3c6..135e0a030 100644 --- a/app/models/tag.rb +++ b/app/models/tag.rb @@ -7,14 +7,14 @@ # name :string default(""), not null # created_at :datetime not null # updated_at :datetime not null -# score :integer # usable :boolean # trendable :boolean # listable :boolean # reviewed_at :datetime # requested_review_at :datetime # last_status_at :datetime -# last_trend_at :datetime +# max_score :float +# max_score_at :datetime # class Tag < ApplicationRecord diff --git a/app/models/trending_tags.rb b/app/models/trending_tags.rb index e4ce988c1..e1b92b175 100644 --- a/app/models/trending_tags.rb +++ b/app/models/trending_tags.rb @@ -7,6 +7,8 @@ class TrendingTags THRESHOLD = 5 LIMIT = 10 REVIEW_THRESHOLD = 3 + MAX_SCORE_COOLDOWN = 3.days.freeze + MAX_SCORE_HALFLIFE = 6.hours.freeze class << self include Redisable @@ -16,14 +18,75 @@ class TrendingTags increment_historical_use!(tag.id, at_time) increment_unique_use!(tag.id, account.id, at_time) - increment_vote!(tag, at_time) + increment_use!(tag.id, at_time) tag.update(last_status_at: Time.now.utc) if tag.last_status_at.nil? || tag.last_status_at < 12.hours.ago - tag.update(last_trend_at: Time.now.utc) if trending?(tag) && (tag.last_trend_at.nil? || tag.last_trend_at < 12.hours.ago) + end + + def update!(at_time = Time.now.utc) + tag_ids = redis.smembers("#{KEY}:used:#{at_time.beginning_of_day.to_i}") + redis.zrange(KEY, 0, -1) + tags = Tag.where(id: tag_ids.uniq) + + # First pass to calculate scores and update the set + + tags.each do |tag| + expected = redis.pfcount("activity:tags:#{tag.id}:#{(at_time - 1.day).beginning_of_day.to_i}:accounts").to_f + expected = 1.0 if expected.zero? + observed = redis.pfcount("activity:tags:#{tag.id}:#{at_time.beginning_of_day.to_i}:accounts").to_f + max_time = tag.max_score_at + max_score = tag.max_score + max_score = 0 if max_time.nil? || max_time < (at_time - MAX_SCORE_COOLDOWN) + + score = begin + if expected > observed || observed < THRESHOLD + 0 + else + ((observed - expected)**2) / expected + end + end + + if score > max_score + max_score = score + max_time = at_time + + # Not interested in triggering any callbacks for this + tag.update_columns(max_score: max_score, max_score_at: max_time) + end + + decaying_score = max_score * (0.5**((at_time.to_f - max_time.to_f) / MAX_SCORE_HALFLIFE.to_f)) + + if decaying_score.zero? + redis.zrem(KEY, tag.id) + else + redis.zadd(KEY, decaying_score, tag.id) + end + end + + users_for_review = User.staff.includes(:account).to_a.select(&:allows_trending_tag_emails?) + + # Second pass to notify about previously unreviewed trends + + tags.each do |tag| + current_rank = redis.zrevrank(KEY, tag.id) + needs_review_notification = tag.requires_review? && !tag.requested_review? + rank_passes_threshold = current_rank.present? && current_rank <= REVIEW_THRESHOLD + + next unless !tag.trendable? && rank_passes_threshold && needs_review_notification + + tag.touch(:requested_review_at) + + users_for_review.each do |user| + AdminMailer.new_trending_tag(user.account, tag).deliver_later! + end + end + + # Trim older items + + redis.zremrangebyrank(KEY, 0, -(LIMIT + 1)) end def get(limit, filtered: true) - tag_ids = redis.zrevrange("#{KEY}:#{Time.now.utc.beginning_of_day.to_i}", 0, LIMIT - 1).map(&:to_i) + tag_ids = redis.zrevrange(KEY, 0, LIMIT - 1).map(&:to_i) tags = Tag.where(id: tag_ids) tags = tags.where(trendable: true) if filtered @@ -33,8 +96,8 @@ class TrendingTags end def trending?(tag) - rank = redis.zrevrank("#{KEY}:#{Time.now.utc.beginning_of_day.to_i}", tag.id) - rank.present? && rank <= LIMIT + rank = redis.zrevrank(KEY, tag.id) + rank.present? && rank < LIMIT end private @@ -51,31 +114,10 @@ class TrendingTags redis.expire(key, EXPIRE_HISTORY_AFTER) end - def increment_vote!(tag, at_time) - key = "#{KEY}:#{at_time.beginning_of_day.to_i}" - expected = redis.pfcount("activity:tags:#{tag.id}:#{(at_time - 1.day).beginning_of_day.to_i}:accounts").to_f - expected = 1.0 if expected.zero? - observed = redis.pfcount("activity:tags:#{tag.id}:#{at_time.beginning_of_day.to_i}:accounts").to_f - - if expected > observed || observed < THRESHOLD - redis.zrem(key, tag.id) - else - score = ((observed - expected)**2) / expected - old_rank = redis.zrevrank(key, tag.id) - - redis.zadd(key, score, tag.id) - request_review!(tag) if (old_rank.nil? || old_rank > REVIEW_THRESHOLD) && redis.zrevrank(key, tag.id) <= REVIEW_THRESHOLD && !tag.trendable? && tag.requires_review? && !tag.requested_review? - end - - redis.expire(key, EXPIRE_TRENDS_AFTER) - end - - def request_review!(tag) - return unless Setting.trends - - tag.touch(:requested_review_at) - - User.staff.includes(:account).find_each { |u| AdminMailer.new_trending_tag(u.account, tag).deliver_later! if u.allows_trending_tag_emails? } + def increment_use!(tag_id, at_time) + key = "#{KEY}:used:#{at_time.beginning_of_day.to_i}" + redis.sadd(key, tag_id) + redis.expire(key, EXPIRE_HISTORY_AFTER) end end end diff --git a/app/workers/scheduler/trending_tags_scheduler.rb b/app/workers/scheduler/trending_tags_scheduler.rb new file mode 100644 index 000000000..77f0d5747 --- /dev/null +++ b/app/workers/scheduler/trending_tags_scheduler.rb @@ -0,0 +1,11 @@ +# frozen_string_literal: true + +class Scheduler::TrendingTagsScheduler + include Sidekiq::Worker + + sidekiq_options unique: :until_executed, retry: 0 + + def perform + TrendingTags.update! if Setting.trends + end +end diff --git a/config/sidekiq.yml b/config/sidekiq.yml index 6ebe450b0..5de25de23 100644 --- a/config/sidekiq.yml +++ b/config/sidekiq.yml @@ -9,6 +9,9 @@ scheduled_statuses_scheduler: every: '5m' class: Scheduler::ScheduledStatusesScheduler + trending_tags_scheduler: + every: '5m' + class: Scheduler::TrendingTagsScheduler media_cleanup_scheduler: cron: '<%= Random.rand(0..59) %> <%= Random.rand(3..5) %> * * *' class: Scheduler::MediaCleanupScheduler diff --git a/db/migrate/20190901035623_add_max_score_to_tags.rb b/db/migrate/20190901035623_add_max_score_to_tags.rb new file mode 100644 index 000000000..f936e9871 --- /dev/null +++ b/db/migrate/20190901035623_add_max_score_to_tags.rb @@ -0,0 +1,6 @@ +class AddMaxScoreToTags < ActiveRecord::Migration[5.2] + def change + add_column :tags, :max_score, :float + add_column :tags, :max_score_at, :datetime + end +end diff --git a/db/post_migrate/20190901040524_remove_score_from_tags.rb b/db/post_migrate/20190901040524_remove_score_from_tags.rb new file mode 100644 index 000000000..a1112700b --- /dev/null +++ b/db/post_migrate/20190901040524_remove_score_from_tags.rb @@ -0,0 +1,12 @@ +# frozen_string_literal: true + +class RemoveScoreFromTags < ActiveRecord::Migration[5.2] + disable_ddl_transaction! + + def change + safety_assured do + remove_column :tags, :score, :int + remove_column :tags, :last_trend_at, :datetime + end + end +end diff --git a/db/schema.rb b/db/schema.rb index 482bca367..5576f70bf 100644 --- a/db/schema.rb +++ b/db/schema.rb @@ -10,7 +10,7 @@ # # It's strongly recommended that you check this file into your version control system. -ActiveRecord::Schema.define(version: 2019_08_23_221802) do +ActiveRecord::Schema.define(version: 2019_09_01_040524) do # These are extensions that must be enabled in order to support this database enable_extension "plpgsql" @@ -664,14 +664,14 @@ ActiveRecord::Schema.define(version: 2019_08_23_221802) do t.string "name", default: "", null: false t.datetime "created_at", null: false t.datetime "updated_at", null: false - t.integer "score" t.boolean "usable" t.boolean "trendable" t.boolean "listable" t.datetime "reviewed_at" t.datetime "requested_review_at" t.datetime "last_status_at" - t.datetime "last_trend_at" + t.float "max_score" + t.datetime "max_score_at" t.index "lower((name)::text)", name: "index_tags_on_name_lower", unique: true end diff --git a/spec/models/trending_tags_spec.rb b/spec/models/trending_tags_spec.rb new file mode 100644 index 000000000..b6122c994 --- /dev/null +++ b/spec/models/trending_tags_spec.rb @@ -0,0 +1,68 @@ +require 'rails_helper' + +RSpec.describe TrendingTags do + describe '.record_use!' do + pending + end + + describe '.update!' do + let!(:at_time) { Time.now.utc } + let!(:tag1) { Fabricate(:tag, name: 'Catstodon') } + let!(:tag2) { Fabricate(:tag, name: 'DogsOfMastodon') } + let!(:tag3) { Fabricate(:tag, name: 'OCs') } + + before do + allow(Redis.current).to receive(:pfcount) do |key| + case key + when "activity:tags:#{tag1.id}:#{(at_time - 1.day).beginning_of_day.to_i}:accounts" + 2 + when "activity:tags:#{tag1.id}:#{at_time.beginning_of_day.to_i}:accounts" + 16 + when "activity:tags:#{tag2.id}:#{(at_time - 1.day).beginning_of_day.to_i}:accounts" + 0 + when "activity:tags:#{tag2.id}:#{at_time.beginning_of_day.to_i}:accounts" + 4 + when "activity:tags:#{tag3.id}:#{(at_time - 1.day).beginning_of_day.to_i}:accounts" + 13 + end + end + + Redis.current.zadd('trending_tags', 0.9, tag3.id) + Redis.current.sadd("trending_tags:used:#{at_time.beginning_of_day.to_i}", [tag1.id, tag2.id]) + + tag3.update(max_score: 0.9, max_score_at: (at_time - 1.day).beginning_of_day + 12.hours) + + described_class.update!(at_time) + end + + it 'calculates and re-calculates scores' do + expect(described_class.get(10, filtered: false)).to eq [tag1, tag3] + end + + it 'omits hashtags below threshold' do + expect(described_class.get(10, filtered: false)).to_not include(tag2) + end + + it 'decays scores' do + expect(Redis.current.zscore('trending_tags', tag3.id)).to be < 0.9 + end + end + + describe '.trending?' do + let(:tag) { Fabricate(:tag) } + + before do + 10.times { |i| Redis.current.zadd('trending_tags', i + 1, Fabricate(:tag).id) } + end + + it 'returns true if the hashtag is within limit' do + Redis.current.zadd('trending_tags', 11, tag.id) + expect(described_class.trending?(tag)).to be true + end + + it 'returns false if the hashtag is outside the limit' do + Redis.current.zadd('trending_tags', 0, tag.id) + expect(described_class.trending?(tag)).to be false + end + end +end -- cgit