From 6e50134a42cb303e6e42f89f9ddb5aacf83e7a6d Mon Sep 17 00:00:00 2001 From: Eugen Rochko Date: Thu, 25 Nov 2021 13:07:38 +0100 Subject: Add trending links (#16917) * Add trending links * Add overriding specific links trendability * Add link type to preview cards and only trend articles Change trends review notifications from being sent every 5 minutes to being sent every 2 hours Change threshold from 5 unique accounts to 15 unique accounts * Fix tests --- app/chewy/tags_index.rb | 2 +- app/controllers/admin/dashboard_controller.rb | 2 +- app/controllers/admin/tags_controller.rb | 76 +----------- .../links/preview_card_providers_controller.rb | 41 +++++++ app/controllers/admin/trends/links_controller.rb | 45 ++++++++ app/controllers/admin/trends/tags_controller.rb | 41 +++++++ .../api/v1/admin/dimensions_controller.rb | 3 +- .../api/v1/admin/measures_controller.rb | 3 +- .../api/v1/admin/trends/tags_controller.rb | 16 +++ app/controllers/api/v1/admin/trends_controller.rb | 16 --- app/controllers/api/v1/trends/links_controller.rb | 21 ++++ app/controllers/api/v1/trends/tags_controller.rb | 21 ++++ app/controllers/api/v1/trends_controller.rb | 15 --- app/helpers/admin/filter_helper.rb | 2 + app/helpers/languages_helper.rb | 94 +++++++++++++++ app/helpers/settings_helper.rb | 89 +------------- .../mastodon/components/admin/Counter.js | 5 +- .../mastodon/components/admin/Dimension.js | 5 +- app/javascript/mastodon/components/admin/Trends.js | 2 +- app/javascript/styles/mastodon/accounts.scss | 16 +++ app/javascript/styles/mastodon/dashboard.scss | 10 ++ app/lib/activitypub/activity.rb | 2 - app/lib/activitypub/activity/announce.rb | 5 +- app/lib/activitypub/activity/create.rb | 7 +- app/lib/admin/metrics/dimension.rb | 9 +- app/lib/admin/metrics/dimension/base_dimension.rb | 13 ++- .../admin/metrics/dimension/languages_dimension.rb | 4 +- .../metrics/dimension/tag_languages_dimension.rb | 36 ++++++ .../metrics/dimension/tag_servers_dimension.rb | 35 ++++++ app/lib/admin/metrics/measure.rb | 10 +- .../admin/metrics/measure/active_users_measure.rb | 4 +- app/lib/admin/metrics/measure/base_measure.rb | 15 ++- .../admin/metrics/measure/interactions_measure.rb | 4 +- .../admin/metrics/measure/tag_accounts_measure.rb | 41 +++++++ .../admin/metrics/measure/tag_servers_measure.rb | 47 ++++++++ app/lib/admin/metrics/measure/tag_uses_measure.rb | 41 +++++++ app/lib/link_details_extractor.rb | 49 +++++++- app/mailers/admin_mailer.rb | 22 +++- app/models/account_statuses_cleanup_policy.rb | 4 +- app/models/form/preview_card_batch.rb | 65 +++++++++++ app/models/form/preview_card_provider_batch.rb | 33 ++++++ app/models/form/tag_batch.rb | 8 +- app/models/preview_card.rb | 42 ++++++- app/models/preview_card_filter.rb | 53 +++++++++ app/models/preview_card_provider.rb | 57 +++++++++ app/models/preview_card_provider_filter.rb | 49 ++++++++ app/models/tag.rb | 23 +--- app/models/tag_filter.rb | 56 +++++---- app/models/trending_tags.rb | 128 --------------------- app/models/trends.rb | 27 +++++ app/models/trends/base.rb | 80 +++++++++++++ app/models/trends/history.rb | 98 ++++++++++++++++ app/models/trends/links.rb | 117 +++++++++++++++++++ app/models/trends/tags.rb | 111 ++++++++++++++++++ app/policies/preview_card_policy.rb | 11 ++ app/policies/preview_card_provider_policy.rb | 11 ++ app/serializers/rest/trends/link_serializer.rb | 5 + app/services/fetch_link_card_service.rb | 3 +- app/services/post_status_service.rb | 3 +- app/services/process_hashtags_service.rb | 2 +- app/services/reblog_service.rb | 13 +-- app/views/admin/dashboard/index.html.haml | 2 +- app/views/admin/tags/_tag.html.haml | 19 --- app/views/admin/tags/index.html.haml | 74 ------------ app/views/admin/tags/show.html.haml | 68 +++++++---- .../admin/trends/links/_preview_card.html.haml | 30 +++++ app/views/admin/trends/links/index.html.haml | 41 +++++++ .../_preview_card_provider.html.haml | 16 +++ .../links/preview_card_providers/index.html.haml | 43 +++++++ app/views/admin/trends/tags/_tag.html.haml | 24 ++++ app/views/admin/trends/tags/index.html.haml | 38 ++++++ app/views/admin_mailer/new_trending_links.text.erb | 16 +++ app/views/admin_mailer/new_trending_tag.text.erb | 5 - app/views/admin_mailer/new_trending_tags.text.erb | 16 +++ app/views/application/_sidebar.html.haml | 2 +- app/workers/scheduler/trending_tags_scheduler.rb | 11 -- app/workers/scheduler/trends/refresh_scheduler.rb | 11 ++ .../trends/review_notifications_scheduler.rb | 11 ++ 78 files changed, 1743 insertions(+), 552 deletions(-) create mode 100644 app/controllers/admin/trends/links/preview_card_providers_controller.rb create mode 100644 app/controllers/admin/trends/links_controller.rb create mode 100644 app/controllers/admin/trends/tags_controller.rb create mode 100644 app/controllers/api/v1/admin/trends/tags_controller.rb delete mode 100644 app/controllers/api/v1/admin/trends_controller.rb create mode 100644 app/controllers/api/v1/trends/links_controller.rb create mode 100644 app/controllers/api/v1/trends/tags_controller.rb delete mode 100644 app/controllers/api/v1/trends_controller.rb create mode 100644 app/helpers/languages_helper.rb create mode 100644 app/lib/admin/metrics/dimension/tag_languages_dimension.rb create mode 100644 app/lib/admin/metrics/dimension/tag_servers_dimension.rb create mode 100644 app/lib/admin/metrics/measure/tag_accounts_measure.rb create mode 100644 app/lib/admin/metrics/measure/tag_servers_measure.rb create mode 100644 app/lib/admin/metrics/measure/tag_uses_measure.rb create mode 100644 app/models/form/preview_card_batch.rb create mode 100644 app/models/form/preview_card_provider_batch.rb create mode 100644 app/models/preview_card_filter.rb create mode 100644 app/models/preview_card_provider.rb create mode 100644 app/models/preview_card_provider_filter.rb delete mode 100644 app/models/trending_tags.rb create mode 100644 app/models/trends.rb create mode 100644 app/models/trends/base.rb create mode 100644 app/models/trends/history.rb create mode 100644 app/models/trends/links.rb create mode 100644 app/models/trends/tags.rb create mode 100644 app/policies/preview_card_policy.rb create mode 100644 app/policies/preview_card_provider_policy.rb create mode 100644 app/serializers/rest/trends/link_serializer.rb delete mode 100644 app/views/admin/tags/_tag.html.haml delete mode 100644 app/views/admin/tags/index.html.haml create mode 100644 app/views/admin/trends/links/_preview_card.html.haml create mode 100644 app/views/admin/trends/links/index.html.haml create mode 100644 app/views/admin/trends/links/preview_card_providers/_preview_card_provider.html.haml create mode 100644 app/views/admin/trends/links/preview_card_providers/index.html.haml create mode 100644 app/views/admin/trends/tags/_tag.html.haml create mode 100644 app/views/admin/trends/tags/index.html.haml create mode 100644 app/views/admin_mailer/new_trending_links.text.erb delete mode 100644 app/views/admin_mailer/new_trending_tag.text.erb create mode 100644 app/views/admin_mailer/new_trending_tags.text.erb delete mode 100644 app/workers/scheduler/trending_tags_scheduler.rb create mode 100644 app/workers/scheduler/trends/refresh_scheduler.rb create mode 100644 app/workers/scheduler/trends/review_notifications_scheduler.rb (limited to 'app') diff --git a/app/chewy/tags_index.rb b/app/chewy/tags_index.rb index f811a8d67..f9db2b03a 100644 --- a/app/chewy/tags_index.rb +++ b/app/chewy/tags_index.rb @@ -31,7 +31,7 @@ class TagsIndex < Chewy::Index end field :reviewed, type: 'boolean', value: ->(tag) { tag.reviewed? } - field :usage, type: 'long', value: ->(tag) { tag.history.reduce(0) { |total, day| total + day[:accounts].to_i } } + field :usage, type: 'long', value: ->(tag) { tag.history.reduce(0) { |total, day| total + day.accounts } } field :last_status_at, type: 'date', value: ->(tag) { tag.last_status_at || tag.created_at } end end diff --git a/app/controllers/admin/dashboard_controller.rb b/app/controllers/admin/dashboard_controller.rb index cbfff2707..f0a935411 100644 --- a/app/controllers/admin/dashboard_controller.rb +++ b/app/controllers/admin/dashboard_controller.rb @@ -4,7 +4,7 @@ module Admin class DashboardController < BaseController def index @system_checks = Admin::SystemCheck.perform - @time_period = (1.month.ago.to_date...Time.now.utc.to_date) + @time_period = (29.days.ago.to_date...Time.now.utc.to_date) @pending_users_count = User.pending.count @pending_reports_count = Report.unresolved.count @pending_tags_count = Tag.pending_review.count diff --git a/app/controllers/admin/tags_controller.rb b/app/controllers/admin/tags_controller.rb index eed4feea2..749e2f144 100644 --- a/app/controllers/admin/tags_controller.rb +++ b/app/controllers/admin/tags_controller.rb @@ -2,38 +2,12 @@ module Admin class TagsController < BaseController - before_action :set_tag, except: [:index, :batch, :approve_all, :reject_all] - before_action :set_usage_by_domain, except: [:index, :batch, :approve_all, :reject_all] - before_action :set_counters, except: [:index, :batch, :approve_all, :reject_all] - - def index - authorize :tag, :index? - - @tags = filtered_tags.page(params[:page]) - @form = Form::TagBatch.new - end - - def batch - @form = Form::TagBatch.new(form_tag_batch_params.merge(current_account: current_account, action: action_from_button)) - @form.save - rescue ActionController::ParameterMissing - flash[:alert] = I18n.t('admin.accounts.no_account_selected') - ensure - redirect_to admin_tags_path(filter_params) - end - - def approve_all - Form::TagBatch.new(current_account: current_account, tag_ids: Tag.pending_review.pluck(:id), action: 'approve').save - redirect_to admin_tags_path(filter_params) - end - - def reject_all - Form::TagBatch.new(current_account: current_account, tag_ids: Tag.pending_review.pluck(:id), action: 'reject').save - redirect_to admin_tags_path(filter_params) - end + before_action :set_tag def show authorize @tag, :show? + + @time_period = (6.days.ago.to_date...Time.now.utc.to_date) end def update @@ -52,52 +26,8 @@ module Admin @tag = Tag.find(params[:id]) end - def set_usage_by_domain - @usage_by_domain = @tag.statuses - .with_public_visibility - .excluding_silenced_accounts - .where(Status.arel_table[:id].gteq(Mastodon::Snowflake.id_at(Time.now.utc.beginning_of_day))) - .joins(:account) - .group('accounts.domain') - .reorder(statuses_count: :desc) - .pluck(Arel.sql('accounts.domain, count(*) AS statuses_count')) - end - - def set_counters - @accounts_today = @tag.history.first[:accounts] - @accounts_week = Redis.current.pfcount(*current_week_days.map { |day| "activity:tags:#{@tag.id}:#{day}:accounts" }) - end - - def filtered_tags - TagFilter.new(filter_params).results - end - - def filter_params - params.slice(:page, *TagFilter::KEYS).permit(:page, *TagFilter::KEYS) - end - def tag_params params.require(:tag).permit(:name, :trendable, :usable, :listable) end - - def current_week_days - now = Time.now.utc.beginning_of_day.to_date - - (Date.commercial(now.cwyear, now.cweek)..now).map do |date| - date.to_time(:utc).beginning_of_day.to_i - end - end - - def form_tag_batch_params - params.require(:form_tag_batch).permit(:action, tag_ids: []) - end - - def action_from_button - if params[:approve] - 'approve' - elsif params[:reject] - 'reject' - end - end end end diff --git a/app/controllers/admin/trends/links/preview_card_providers_controller.rb b/app/controllers/admin/trends/links/preview_card_providers_controller.rb new file mode 100644 index 000000000..2c26e03f3 --- /dev/null +++ b/app/controllers/admin/trends/links/preview_card_providers_controller.rb @@ -0,0 +1,41 @@ +# frozen_string_literal: true + +class Admin::Trends::Links::PreviewCardProvidersController < Admin::BaseController + def index + authorize :preview_card_provider, :index? + + @preview_card_providers = filtered_preview_card_providers.page(params[:page]) + @form = Form::PreviewCardProviderBatch.new + end + + def batch + @form = Form::PreviewCardProviderBatch.new(form_preview_card_provider_batch_params.merge(current_account: current_account, action: action_from_button)) + @form.save + rescue ActionController::ParameterMissing + flash[:alert] = I18n.t('admin.accounts.no_account_selected') + ensure + redirect_to admin_trends_links_preview_card_providers_path(filter_params) + end + + private + + def filtered_preview_card_providers + PreviewCardProviderFilter.new(filter_params).results + end + + def filter_params + params.slice(:page, *PreviewCardProviderFilter::KEYS).permit(:page, *PreviewCardProviderFilter::KEYS) + end + + def form_preview_card_provider_batch_params + params.require(:form_preview_card_provider_batch).permit(:action, preview_card_provider_ids: []) + end + + def action_from_button + if params[:approve] + 'approve' + elsif params[:reject] + 'reject' + end + end +end diff --git a/app/controllers/admin/trends/links_controller.rb b/app/controllers/admin/trends/links_controller.rb new file mode 100644 index 000000000..619b37deb --- /dev/null +++ b/app/controllers/admin/trends/links_controller.rb @@ -0,0 +1,45 @@ +# frozen_string_literal: true + +class Admin::Trends::LinksController < Admin::BaseController + def index + authorize :preview_card, :index? + + @preview_cards = filtered_preview_cards.page(params[:page]) + @form = Form::PreviewCardBatch.new + end + + def batch + @form = Form::PreviewCardBatch.new(form_preview_card_batch_params.merge(current_account: current_account, action: action_from_button)) + @form.save + rescue ActionController::ParameterMissing + flash[:alert] = I18n.t('admin.accounts.no_account_selected') + ensure + redirect_to admin_trends_links_path(filter_params) + end + + private + + def filtered_preview_cards + PreviewCardFilter.new(filter_params.with_defaults(trending: 'all')).results + end + + def filter_params + params.slice(:page, *PreviewCardFilter::KEYS).permit(:page, *PreviewCardFilter::KEYS) + end + + def form_preview_card_batch_params + params.require(:form_preview_card_batch).permit(:action, preview_card_ids: []) + end + + def action_from_button + if params[:approve] + 'approve' + elsif params[:approve_all] + 'approve_all' + elsif params[:reject] + 'reject' + elsif params[:reject_all] + 'reject_all' + end + end +end diff --git a/app/controllers/admin/trends/tags_controller.rb b/app/controllers/admin/trends/tags_controller.rb new file mode 100644 index 000000000..91ff33d40 --- /dev/null +++ b/app/controllers/admin/trends/tags_controller.rb @@ -0,0 +1,41 @@ +# frozen_string_literal: true + +class Admin::Trends::TagsController < Admin::BaseController + def index + authorize :tag, :index? + + @tags = filtered_tags.page(params[:page]) + @form = Form::TagBatch.new + end + + def batch + @form = Form::TagBatch.new(form_tag_batch_params.merge(current_account: current_account, action: action_from_button)) + @form.save + rescue ActionController::ParameterMissing + flash[:alert] = I18n.t('admin.accounts.no_account_selected') + ensure + redirect_to admin_trends_tags_path(filter_params) + end + + private + + def filtered_tags + TagFilter.new(filter_params).results + end + + def filter_params + params.slice(:page, *TagFilter::KEYS).permit(:page, *TagFilter::KEYS) + end + + def form_tag_batch_params + params.require(:form_tag_batch).permit(:action, tag_ids: []) + end + + def action_from_button + if params[:approve] + 'approve' + elsif params[:reject] + 'reject' + end + end +end diff --git a/app/controllers/api/v1/admin/dimensions_controller.rb b/app/controllers/api/v1/admin/dimensions_controller.rb index 170596d27..5e8f0f89f 100644 --- a/app/controllers/api/v1/admin/dimensions_controller.rb +++ b/app/controllers/api/v1/admin/dimensions_controller.rb @@ -17,7 +17,8 @@ class Api::V1::Admin::DimensionsController < Api::BaseController params[:keys], params[:start_at], params[:end_at], - params[:limit] + params[:limit], + params ) end end diff --git a/app/controllers/api/v1/admin/measures_controller.rb b/app/controllers/api/v1/admin/measures_controller.rb index a3ac6fe85..f28191753 100644 --- a/app/controllers/api/v1/admin/measures_controller.rb +++ b/app/controllers/api/v1/admin/measures_controller.rb @@ -16,7 +16,8 @@ class Api::V1::Admin::MeasuresController < Api::BaseController @measures = Admin::Metrics::Measure.retrieve( params[:keys], params[:start_at], - params[:end_at] + params[:end_at], + params ) end end diff --git a/app/controllers/api/v1/admin/trends/tags_controller.rb b/app/controllers/api/v1/admin/trends/tags_controller.rb new file mode 100644 index 000000000..3653d1dd1 --- /dev/null +++ b/app/controllers/api/v1/admin/trends/tags_controller.rb @@ -0,0 +1,16 @@ +# frozen_string_literal: true + +class Api::V1::Admin::Trends::TagsController < Api::BaseController + before_action :require_staff! + before_action :set_tags + + def index + render json: @tags, each_serializer: REST::Admin::TagSerializer + end + + private + + def set_tags + @tags = Trends.tags.get(false, limit_param(10)) + end +end diff --git a/app/controllers/api/v1/admin/trends_controller.rb b/app/controllers/api/v1/admin/trends_controller.rb deleted file mode 100644 index e32ab5d2c..000000000 --- a/app/controllers/api/v1/admin/trends_controller.rb +++ /dev/null @@ -1,16 +0,0 @@ -# frozen_string_literal: true - -class Api::V1::Admin::TrendsController < Api::BaseController - before_action :require_staff! - before_action :set_trends - - def index - render json: @trends, each_serializer: REST::Admin::TagSerializer - end - - private - - def set_trends - @trends = TrendingTags.get(10, filtered: false) - end -end diff --git a/app/controllers/api/v1/trends/links_controller.rb b/app/controllers/api/v1/trends/links_controller.rb new file mode 100644 index 000000000..1c3ab1e1c --- /dev/null +++ b/app/controllers/api/v1/trends/links_controller.rb @@ -0,0 +1,21 @@ +# frozen_string_literal: true + +class Api::V1::Trends::LinksController < Api::BaseController + before_action :set_links + + def index + render json: @links, each_serializer: REST::Trends::LinkSerializer + end + + private + + def set_links + @links = begin + if Setting.trends + Trends.links.get(true, limit_param(10)) + else + [] + end + end + end +end diff --git a/app/controllers/api/v1/trends/tags_controller.rb b/app/controllers/api/v1/trends/tags_controller.rb new file mode 100644 index 000000000..947b53de2 --- /dev/null +++ b/app/controllers/api/v1/trends/tags_controller.rb @@ -0,0 +1,21 @@ +# frozen_string_literal: true + +class Api::V1::Trends::TagsController < Api::BaseController + before_action :set_tags + + def index + render json: @tags, each_serializer: REST::TagSerializer + end + + private + + def set_tags + @tags = begin + if Setting.trends + Trends.tags.get(true, limit_param(10)) + else + [] + end + end + end +end diff --git a/app/controllers/api/v1/trends_controller.rb b/app/controllers/api/v1/trends_controller.rb deleted file mode 100644 index c875e9041..000000000 --- a/app/controllers/api/v1/trends_controller.rb +++ /dev/null @@ -1,15 +0,0 @@ -# frozen_string_literal: true - -class Api::V1::TrendsController < Api::BaseController - before_action :set_tags - - def index - render json: @tags, each_serializer: REST::TagSerializer - end - - private - - def set_tags - @tags = TrendingTags.get(limit_param(10)) - end -end diff --git a/app/helpers/admin/filter_helper.rb b/app/helpers/admin/filter_helper.rb index ba0ca9638..5f69f176a 100644 --- a/app/helpers/admin/filter_helper.rb +++ b/app/helpers/admin/filter_helper.rb @@ -6,6 +6,8 @@ module Admin::FilterHelper CustomEmojiFilter::KEYS, ReportFilter::KEYS, TagFilter::KEYS, + PreviewCardProviderFilter::KEYS, + PreviewCardFilter::KEYS, InstanceFilter::KEYS, InviteFilter::KEYS, RelationshipFilter::KEYS, diff --git a/app/helpers/languages_helper.rb b/app/helpers/languages_helper.rb new file mode 100644 index 000000000..730724208 --- /dev/null +++ b/app/helpers/languages_helper.rb @@ -0,0 +1,94 @@ +# frozen_string_literal: true + +module LanguagesHelper + HUMAN_LOCALES = { + af: 'Afrikaans', + ar: 'العربية', + ast: 'Asturianu', + bg: 'Български', + bn: 'বাংলা', + br: 'Breton', + ca: 'Català', + co: 'Corsu', + cs: 'Čeština', + cy: 'Cymraeg', + da: 'Dansk', + de: 'Deutsch', + el: 'Ελληνικά', + en: 'English', + eo: 'Esperanto', + 'es-AR': 'Español (Argentina)', + 'es-MX': 'Español (México)', + es: 'Español', + et: 'Eesti', + eu: 'Euskara', + fa: 'فارسی', + fi: 'Suomi', + fr: 'Français', + ga: 'Gaeilge', + gd: 'Gàidhlig', + gl: 'Galego', + he: 'עברית', + hi: 'हिन्दी', + hr: 'Hrvatski', + hu: 'Magyar', + hy: 'Հայերեն', + id: 'Bahasa Indonesia', + io: 'Ido', + is: 'Íslenska', + it: 'Italiano', + ja: '日本語', + ka: 'ქართული', + kab: 'Taqbaylit', + kk: 'Қазақша', + kmr: 'Kurmancî', + kn: 'ಕನ್ನಡ', + ko: '한국어', + ku: 'سۆرانی', + lt: 'Lietuvių', + lv: 'Latviešu', + mk: 'Македонски', + ml: 'മലയാളം', + mr: 'मराठी', + ms: 'Bahasa Melayu', + nl: 'Nederlands', + nn: 'Nynorsk', + no: 'Norsk', + oc: 'Occitan', + pl: 'Polski', + 'pt-BR': 'Português (Brasil)', + 'pt-PT': 'Português (Portugal)', + pt: 'Português', + ro: 'Română', + ru: 'Русский', + sa: 'संस्कृतम्', + sc: 'Sardu', + si: 'සිංහල', + sk: 'Slovenčina', + sl: 'Slovenščina', + sq: 'Shqip', + 'sr-Latn': 'Srpski (latinica)', + sr: 'Српски', + sv: 'Svenska', + ta: 'தமிழ்', + te: 'తెలుగు', + th: 'ไทย', + tr: 'Türkçe', + uk: 'Українська', + ur: 'اُردُو', + vi: 'Tiếng Việt', + zgh: 'ⵜⴰⵎⴰⵣⵉⵖⵜ', + 'zh-CN': '简体中文', + 'zh-HK': '繁體中文（香港）', + 'zh-TW': '繁體中文（臺灣）', + zh: '中文', + }.freeze + + def human_locale(locale) + if locale == 'und' + I18n.t('generic.none') + else + HUMAN_LOCALES[locale.to_sym] || locale + end + end +end diff --git a/app/helpers/settings_helper.rb b/app/helpers/settings_helper.rb index ac4c18746..23739d1cd 100644 --- a/app/helpers/settings_helper.rb +++ b/app/helpers/settings_helper.rb @@ -1,95 +1,8 @@ # frozen_string_literal: true module SettingsHelper - HUMAN_LOCALES = { - af: 'Afrikaans', - ar: 'العربية', - ast: 'Asturianu', - bg: 'Български', - bn: 'বাংলা', - br: 'Breton', - ca: 'Català', - co: 'Corsu', - cs: 'Čeština', - cy: 'Cymraeg', - da: 'Dansk', - de: 'Deutsch', - el: 'Ελληνικά', - en: 'English', - eo: 'Esperanto', - 'es-AR': 'Español (Argentina)', - 'es-MX': 'Español (México)', - es: 'Español', - et: 'Eesti', - eu: 'Euskara', - fa: 'فارسی', - fi: 'Suomi', - fr: 'Français', - ga: 'Gaeilge', - gd: 'Gàidhlig', - gl: 'Galego', - he: 'עברית', - hi: 'हिन्दी', - hr: 'Hrvatski', - hu: 'Magyar', - hy: 'Հայերեն', - id: 'Bahasa Indonesia', - io: 'Ido', - is: 'Íslenska', - it: 'Italiano', - ja: '日本語', - ka: 'ქართული', - kab: 'Taqbaylit', - kk: 'Қазақша', - kmr: 'Kurmancî', - kn: 'ಕನ್ನಡ', - ko: '한국어', - ku: 'سۆرانی', - lt: 'Lietuvių', - lv: 'Latviešu', - mk: 'Македонски', - ml: 'മലയാളം', - mr: 'मराठी', - ms: 'Bahasa Melayu', - nl: 'Nederlands', - nn: 'Nynorsk', - no: 'Norsk', - oc: 'Occitan', - pl: 'Polski', - 'pt-BR': 'Português (Brasil)', - 'pt-PT': 'Português (Portugal)', - pt: 'Português', - ro: 'Română', - ru: 'Русский', - sa: 'संस्कृतम्', - sc: 'Sardu', - si: 'සිංහල', - sk: 'Slovenčina', - sl: 'Slovenščina', - sq: 'Shqip', - 'sr-Latn': 'Srpski (latinica)', - sr: 'Српски', - sv: 'Svenska', - ta: 'தமிழ்', - te: 'తెలుగు', - th: 'ไทย', - tr: 'Türkçe', - uk: 'Українська', - ur: 'اُردُو', - vi: 'Tiếng Việt', - zgh: 'ⵜⴰⵎⴰⵣⵉⵖⵜ', - 'zh-CN': '简体中文', - 'zh-HK': '繁體中文（香港）', - 'zh-TW': '繁體中文（臺灣）', - zh: '中文', - }.freeze - - def human_locale(locale) - HUMAN_LOCALES[locale] - end - def filterable_languages - LanguageDetector.instance.language_names.select(&HUMAN_LOCALES.method(:key?)) + LanguageDetector.instance.language_names.select(&LanguagesHelper::HUMAN_LOCALES.method(:key?)) end def hash_to_object(hash) diff --git a/app/javascript/mastodon/components/admin/Counter.js b/app/javascript/mastodon/components/admin/Counter.js index cda572dce..047e864b2 100644 --- a/app/javascript/mastodon/components/admin/Counter.js +++ b/app/javascript/mastodon/components/admin/Counter.js @@ -32,6 +32,7 @@ export default class Counter extends React.PureComponent { end_at: PropTypes.string.isRequired, label: PropTypes.string.isRequired, href: PropTypes.string, + params: PropTypes.object, }; state = { @@ -40,9 +41,9 @@ export default class Counter extends React.PureComponent { }; componentDidMount () { - const { measure, start_at, end_at } = this.props; + const { measure, start_at, end_at, params } = this.props; - api().post('/api/v1/admin/measures', { keys: [measure], start_at, end_at }).then(res => { + api().post('/api/v1/admin/measures', { keys: [measure], start_at, end_at, [measure]: params }).then(res => { this.setState({ loading: false, data: res.data, diff --git a/app/javascript/mastodon/components/admin/Dimension.js b/app/javascript/mastodon/components/admin/Dimension.js index ac6dbd1c7..977c8208d 100644 --- a/app/javascript/mastodon/components/admin/Dimension.js +++ b/app/javascript/mastodon/components/admin/Dimension.js @@ -13,6 +13,7 @@ export default class Dimension extends React.PureComponent { end_at: PropTypes.string.isRequired, limit: PropTypes.number.isRequired, label: PropTypes.string.isRequired, + params: PropTypes.object, }; state = { @@ -21,9 +22,9 @@ export default class Dimension extends React.PureComponent { }; componentDidMount () { - const { start_at, end_at, dimension, limit } = this.props; + const { start_at, end_at, dimension, limit, params } = this.props; - api().post('/api/v1/admin/dimensions', { keys: [dimension], start_at, end_at, limit }).then(res => { + api().post('/api/v1/admin/dimensions', { keys: [dimension], start_at, end_at, limit, [dimension]: params }).then(res => { this.setState({ loading: false, data: res.data, diff --git a/app/javascript/mastodon/components/admin/Trends.js b/app/javascript/mastodon/components/admin/Trends.js index 46307a28a..635bdf37d 100644 --- a/app/javascript/mastodon/components/admin/Trends.js +++ b/app/javascript/mastodon/components/admin/Trends.js @@ -19,7 +19,7 @@ export default class Trends extends React.PureComponent { componentDidMount () { const { limit } = this.props; - api().get('/api/v1/admin/trends', { params: { limit } }).then(res => { + api().get('/api/v1/admin/trends/tags', { params: { limit } }).then(res => { this.setState({ loading: false, data: res.data, diff --git a/app/javascript/styles/mastodon/accounts.scss b/app/javascript/styles/mastodon/accounts.scss index 2c78e81be..b8a6c8018 100644 --- a/app/javascript/styles/mastodon/accounts.scss +++ b/app/javascript/styles/mastodon/accounts.scss @@ -325,3 +325,19 @@ margin-top: 10px; } } + +.batch-table__row--muted .pending-account__header { + &, + a, + strong { + color: lighten($ui-base-color, 26%); + } +} + +.batch-table__row--attention .pending-account__header { + &, + a, + strong { + color: $gold-star; + } +} diff --git a/app/javascript/styles/mastodon/dashboard.scss b/app/javascript/styles/mastodon/dashboard.scss index 5e900e8c5..0a881bc10 100644 --- a/app/javascript/styles/mastodon/dashboard.scss +++ b/app/javascript/styles/mastodon/dashboard.scss @@ -100,6 +100,16 @@ transition: all 200ms ease-out; } + &.positive { + background: lighten($ui-base-color, 4%); + color: $valid-value-color; + } + + &.negative { + background: lighten($ui-base-color, 4%); + color: $error-value-color; + } + span { flex: 1 1 auto; } diff --git a/app/lib/activitypub/activity.rb b/app/lib/activitypub/activity.rb index d2ec122a4..3aeecb4ec 100644 --- a/app/lib/activitypub/activity.rb +++ b/app/lib/activitypub/activity.rb @@ -129,8 +129,6 @@ class ActivityPub::Activity end def crawl_links(status) - return if status.spoiler_text? - # Spread out crawling randomly to avoid DDoSing the link LinkCrawlWorker.perform_in(rand(1..59).seconds, status.id) end diff --git a/app/lib/activitypub/activity/announce.rb b/app/lib/activitypub/activity/announce.rb index 9f778ffb9..6c5d88d18 100644 --- a/app/lib/activitypub/activity/announce.rb +++ b/app/lib/activitypub/activity/announce.rb @@ -22,9 +22,8 @@ class ActivityPub::Activity::Announce < ActivityPub::Activity visibility: visibility_from_audience ) - original_status.tags.each do |tag| - tag.use!(@account) - end + Trends.tags.register(@status) + Trends.links.register(@status) distribute(@status) end diff --git a/app/lib/activitypub/activity/create.rb b/app/lib/activitypub/activity/create.rb index 4c13a80a6..8a0dc9d33 100644 --- a/app/lib/activitypub/activity/create.rb +++ b/app/lib/activitypub/activity/create.rb @@ -164,9 +164,14 @@ class ActivityPub::Activity::Create < ActivityPub::Activity def attach_tags(status) @tags.each do |tag| status.tags << tag - tag.use!(@account, status: status, at_time: status.created_at) if status.public_visibility? + tag.update(last_status_at: status.created_at) if tag.last_status_at.nil? || (tag.last_status_at < status.created_at && tag.last_status_at < 12.hours.ago) end + # If we're processing an old status, this may register tags as being used now + # as opposed to when the status was really published, but this is probably + # not a big deal + Trends.tags.register(status) + @mentions.each do |mention| mention.status = status mention.save diff --git a/app/lib/admin/metrics/dimension.rb b/app/lib/admin/metrics/dimension.rb index 279539f68..d8392ddfc 100644 --- a/app/lib/admin/metrics/dimension.rb +++ b/app/lib/admin/metrics/dimension.rb @@ -7,9 +7,14 @@ class Admin::Metrics::Dimension servers: Admin::Metrics::Dimension::ServersDimension, space_usage: Admin::Metrics::Dimension::SpaceUsageDimension, software_versions: Admin::Metrics::Dimension::SoftwareVersionsDimension, + tag_servers: Admin::Metrics::Dimension::TagServersDimension, + tag_languages: Admin::Metrics::Dimension::TagLanguagesDimension, }.freeze - def self.retrieve(dimension_keys, start_at, end_at, limit) - Array(dimension_keys).map { |key| DIMENSIONS[key.to_sym]&.new(start_at, end_at, limit) }.compact + def self.retrieve(dimension_keys, start_at, end_at, limit, params) + Array(dimension_keys).map do |key| + klass = DIMENSIONS[key.to_sym] + klass&.new(start_at, end_at, limit, klass.with_params? ? params.require(key.to_sym) : nil) + end.compact end end diff --git a/app/lib/admin/metrics/dimension/base_dimension.rb b/app/lib/admin/metrics/dimension/base_dimension.rb index 8ed8d7683..5872c22cb 100644 --- a/app/lib/admin/metrics/dimension/base_dimension.rb +++ b/app/lib/admin/metrics/dimension/base_dimension.rb @@ -1,10 +1,15 @@ # frozen_string_literal: true class Admin::Metrics::Dimension::BaseDimension - def initialize(start_at, end_at, limit) + def self.with_params? + false + end + + def initialize(start_at, end_at, limit, params) @start_at = start_at&.to_datetime @end_at = end_at&.to_datetime @limit = limit&.to_i + @params = params end def key @@ -26,6 +31,10 @@ class Admin::Metrics::Dimension::BaseDimension protected def time_period - (@start_at...@end_at) + (@start_at..@end_at) + end + + def params + raise NotImplementedError end end diff --git a/app/lib/admin/metrics/dimension/languages_dimension.rb b/app/lib/admin/metrics/dimension/languages_dimension.rb index 2d0ac124e..a6aaf5d21 100644 --- a/app/lib/admin/metrics/dimension/languages_dimension.rb +++ b/app/lib/admin/metrics/dimension/languages_dimension.rb @@ -1,6 +1,8 @@ # frozen_string_literal: true class Admin::Metrics::Dimension::LanguagesDimension < Admin::Metrics::Dimension::BaseDimension + include LanguagesHelper + def key 'languages' end @@ -18,6 +20,6 @@ class Admin::Metrics::Dimension::LanguagesDimension < Admin::Metrics::Dimension: rows = ActiveRecord::Base.connection.select_all(sql, nil, [[nil, @start_at], [nil, @end_at], [nil, @limit]]) - rows.map { |row| { key: row['locale'], human_key: SettingsHelper::HUMAN_LOCALES[row['locale'].to_sym], value: row['value'].to_s } } + rows.map { |row| { key: row['locale'], human_key: human_locale(row['locale']), value: row['value'].to_s } } end end diff --git a/app/lib/admin/metrics/dimension/tag_languages_dimension.rb b/app/lib/admin/metrics/dimension/tag_languages_dimension.rb new file mode 100644 index 000000000..1cfa07478 --- /dev/null +++ b/app/lib/admin/metrics/dimension/tag_languages_dimension.rb @@ -0,0 +1,36 @@ +# frozen_string_literal: true + +class Admin::Metrics::Dimension::TagLanguagesDimension < Admin::Metrics::Dimension::BaseDimension + include LanguagesHelper + + def self.with_params? + true + end + + def key + 'tag_languages' + end + + def data + sql = <<-SQL.squish + SELECT COALESCE(statuses.language, 'und') AS language, count(*) AS value + FROM statuses + INNER JOIN statuses_tags ON statuses_tags.status_id = statuses.id + WHERE statuses_tags.tag_id = $1 + AND statuses.id BETWEEN $2 AND $3 + GROUP BY COALESCE(statuses.language, 'und') + ORDER BY count(*) DESC + LIMIT $4 + SQL + + rows = ActiveRecord::Base.connection.select_all(sql, nil, [[nil, params[:id]], [nil, Mastodon::Snowflake.id_at(@start_at, with_random: false)], [nil, Mastodon::Snowflake.id_at(@end_at, with_random: false)], [nil, @limit]]) + + rows.map { |row| { key: row['language'], human_key: human_locale(row['language']), value: row['value'].to_s } } + end + + private + + def params + @params.permit(:id) + end +end diff --git a/app/lib/admin/metrics/dimension/tag_servers_dimension.rb b/app/lib/admin/metrics/dimension/tag_servers_dimension.rb new file mode 100644 index 000000000..12c5980d7 --- /dev/null +++ b/app/lib/admin/metrics/dimension/tag_servers_dimension.rb @@ -0,0 +1,35 @@ +# frozen_string_literal: true + +class Admin::Metrics::Dimension::TagServersDimension < Admin::Metrics::Dimension::BaseDimension + def self.with_params? + true + end + + def key + 'tag_servers' + end + + def data + sql = <<-SQL.squish + SELECT accounts.domain, count(*) AS value + FROM statuses + INNER JOIN accounts ON accounts.id = statuses.account_id + INNER JOIN statuses_tags ON statuses_tags.status_id = statuses.id + WHERE statuses_tags.tag_id = $1 + AND statuses.id BETWEEN $2 AND $3 + GROUP BY accounts.domain + ORDER BY count(*) DESC + LIMIT $4 + SQL + + rows = ActiveRecord::Base.connection.select_all(sql, nil, [[nil, params[:id]], [nil, Mastodon::Snowflake.id_at(@start_at, with_random: false)], [nil, Mastodon::Snowflake.id_at(@end_at, with_random: false)], [nil, @limit]]) + + rows.map { |row| { key: row['domain'] || Rails.configuration.x.local_domain, human_key: row['domain'] || Rails.configuration.x.local_domain, value: row['value'].to_s } } + end + + private + + def params + @params.permit(:id) + end +end diff --git a/app/lib/admin/metrics/measure.rb b/app/lib/admin/metrics/measure.rb index 5cebf0331..a839498a1 100644 --- a/app/lib/admin/metrics/measure.rb +++ b/app/lib/admin/metrics/measure.rb @@ -7,9 +7,15 @@ class Admin::Metrics::Measure interactions: Admin::Metrics::Measure::InteractionsMeasure, opened_reports: Admin::Metrics::Measure::OpenedReportsMeasure, resolved_reports: Admin::Metrics::Measure::ResolvedReportsMeasure, + tag_accounts: Admin::Metrics::Measure::TagAccountsMeasure, + tag_uses: Admin::Metrics::Measure::TagUsesMeasure, + tag_servers: Admin::Metrics::Measure::TagServersMeasure, }.freeze - def self.retrieve(measure_keys, start_at, end_at) - Array(measure_keys).map { |key| MEASURES[key.to_sym]&.new(start_at, end_at) }.compact + def self.retrieve(measure_keys, start_at, end_at, params) + Array(measure_keys).map do |key| + klass = MEASURES[key.to_sym] + klass&.new(start_at, end_at, klass.with_params? ? params.require(key.to_sym) : nil) + end.compact end end diff --git a/app/lib/admin/metrics/measure/active_users_measure.rb b/app/lib/admin/metrics/measure/active_users_measure.rb index ac022eb9d..513189780 100644 --- a/app/lib/admin/metrics/measure/active_users_measure.rb +++ b/app/lib/admin/metrics/measure/active_users_measure.rb @@ -24,10 +24,10 @@ class Admin::Metrics::Measure::ActiveUsersMeasure < Admin::Metrics::Measure::Bas end def time_period - (@start_at.to_date...@end_at.to_date) + (@start_at.to_date..@end_at.to_date) end def previous_time_period - ((@start_at.to_date - length_of_period)...(@end_at.to_date - length_of_period)) + ((@start_at.to_date - length_of_period)..(@end_at.to_date - length_of_period)) end end diff --git a/app/lib/admin/metrics/measure/base_measure.rb b/app/lib/admin/metrics/measure/base_measure.rb index 4c336a69e..0107ffd9c 100644 --- a/app/lib/admin/metrics/measure/base_measure.rb +++ b/app/lib/admin/metrics/measure/base_measure.rb @@ -1,9 +1,14 @@ # frozen_string_literal: true class Admin::Metrics::Measure::BaseMeasure - def initialize(start_at, end_at) + def self.with_params? + false + end + + def initialize(start_at, end_at, params) @start_at = start_at&.to_datetime @end_at = end_at&.to_datetime + @params = params end def key @@ -33,14 +38,18 @@ class Admin::Metrics::Measure::BaseMeasure protected def time_period - (@start_at...@end_at) + (@start_at..@end_at) end def previous_time_period - ((@start_at - length_of_period)...(@end_at - length_of_period)) + ((@start_at - length_of_period)..(@end_at - length_of_period)) end def length_of_period @length_of_period ||= @end_at - @start_at end + + def params + raise NotImplementedError + end end diff --git a/app/lib/admin/metrics/measure/interactions_measure.rb b/app/lib/admin/metrics/measure/interactions_measure.rb index 9a4ef6d63..b928fdb8f 100644 --- a/app/lib/admin/metrics/measure/interactions_measure.rb +++ b/app/lib/admin/metrics/measure/interactions_measure.rb @@ -24,10 +24,10 @@ class Admin::Metrics::Measure::InteractionsMeasure < Admin::Metrics::Measure::Ba end def time_period - (@start_at.to_date...@end_at.to_date) + (@start_at.to_date..@end_at.to_date) end def previous_time_period - ((@start_at.to_date - length_of_period)...(@end_at.to_date - length_of_period)) + ((@start_at.to_date - length_of_period)..(@end_at.to_date - length_of_period)) end end diff --git a/app/lib/admin/metrics/measure/tag_accounts_measure.rb b/app/lib/admin/metrics/measure/tag_accounts_measure.rb new file mode 100644 index 000000000..ef773081b --- /dev/null +++ b/app/lib/admin/metrics/measure/tag_accounts_measure.rb @@ -0,0 +1,41 @@ +# frozen_string_literal: true + +class Admin::Metrics::Measure::TagAccountsMeasure < Admin::Metrics::Measure::BaseMeasure + def self.with_params? + true + end + + def key + 'tag_accounts' + end + + def total + tag.history.aggregate(time_period).accounts + end + + def previous_total + tag.history.aggregate(previous_time_period).accounts + end + + def data + time_period.map { |date| { date: date.to_time(:utc).iso8601, value: tag.history.get(date).accounts.to_s } } + end + + protected + + def tag + @tag ||= Tag.find(params[:id]) + end + + def time_period + (@start_at.to_date..@end_at.to_date) + end + + def previous_time_period + ((@start_at.to_date - length_of_period)..(@end_at.to_date - length_of_period)) + end + + def params + @params.permit(:id) + end +end diff --git a/app/lib/admin/metrics/measure/tag_servers_measure.rb b/app/lib/admin/metrics/measure/tag_servers_measure.rb new file mode 100644 index 000000000..8c3e0551a --- /dev/null +++ b/app/lib/admin/metrics/measure/tag_servers_measure.rb @@ -0,0 +1,47 @@ +# frozen_string_literal: true + +class Admin::Metrics::Measure::TagServersMeasure < Admin::Metrics::Measure::BaseMeasure + def self.with_params? + true + end + + def key + 'tag_servers' + end + + def total + tag.statuses.where('statuses.id BETWEEN ? AND ?', Mastodon::Snowflake.id_at(@start_at, with_random: false), Mastodon::Snowflake.id_at(@end_at, with_random: false)).joins(:account).count('distinct accounts.domain') + end + + def previous_total + tag.statuses.where('statuses.id BETWEEN ? AND ?', Mastodon::Snowflake.id_at(@start_at - length_of_period, with_random: false), Mastodon::Snowflake.id_at(@end_at - length_of_period, with_random: false)).joins(:account).count('distinct accounts.domain') + end + + def data + sql = <<-SQL.squish + SELECT axis.*, ( + SELECT count(*) AS value + FROM statuses + WHERE statuses.id BETWEEN $1 AND $2 + AND date_trunc('day', statuses.created_at)::date = axis.day + ) + FROM ( + SELECT generate_series(date_trunc('day', $3::timestamp)::date, date_trunc('day', $4::timestamp)::date, ('1 day')::interval) AS day + ) as axis + SQL + + rows = ActiveRecord::Base.connection.select_all(sql, nil, [[nil, Mastodon::Snowflake.id_at(@start_at, with_random: false)], [nil, Mastodon::Snowflake.id_at(@end_at, with_random: false)], [nil, @start_at], [nil, @end_at]]) + + rows.map { |row| { date: row['day'], value: row['value'].to_s } } + end + + protected + + def tag + @tag ||= Tag.find(params[:id]) + end + + def params + @params.permit(:id) + end +end diff --git a/app/lib/admin/metrics/measure/tag_uses_measure.rb b/app/lib/admin/metrics/measure/tag_uses_measure.rb new file mode 100644 index 000000000..b7667bc6c --- /dev/null +++ b/app/lib/admin/metrics/measure/tag_uses_measure.rb @@ -0,0 +1,41 @@ +# frozen_string_literal: true + +class Admin::Metrics::Measure::TagUsesMeasure < Admin::Metrics::Measure::BaseMeasure + def self.with_params? + true + end + + def key + 'tag_uses' + end + + def total + tag.history.aggregate(time_period).uses + end + + def previous_total + tag.history.aggregate(previous_time_period).uses + end + + def data + time_period.map { |date| { date: date.to_time(:utc).iso8601, value: tag.history.get(date).uses.to_s } } + end + + protected + + def tag + @tag ||= Tag.find(params[:id]) + end + + def time_period + (@start_at.to_date..@end_at.to_date) + end + + def previous_time_period + ((@start_at.to_date - length_of_period)..(@end_at.to_date - length_of_period)) + end + + def params + @params.permit(:id) + end +end diff --git a/app/lib/link_details_extractor.rb b/app/lib/link_details_extractor.rb index 8b38e8d0c..56ad0717b 100644 --- a/app/lib/link_details_extractor.rb +++ b/app/lib/link_details_extractor.rb @@ -4,6 +4,11 @@ class LinkDetailsExtractor include ActionView::Helpers::TagHelper class StructuredData + SUPPORTED_TYPES = %w( + NewsArticle + WebPage + ).freeze + def initialize(data) @data = data end @@ -16,6 +21,14 @@ class LinkDetailsExtractor json['description'] end + def language + json['inLanguage'] + end + + def type + json['@type'] + end + def image obj = first_of_value(json['image']) @@ -44,6 +57,10 @@ class LinkDetailsExtractor publisher['name'] end + def publisher_logo + publisher.dig('logo', 'url') + end + private def author @@ -58,8 +75,12 @@ class LinkDetailsExtractor arr.is_a?(Array) ? arr.first : arr end + def root_array(root) + root.is_a?(Array) ? root : [root] + end + def json - @json ||= first_of_value(Oj.load(@data)) + @json ||= root_array(Oj.load(@data)).find { |obj| SUPPORTED_TYPES.include?(obj['@type']) } || {} end end @@ -75,6 +96,7 @@ class LinkDetailsExtractor description: description || '', image_remote_url: image, type: type, + link_type: link_type, width: width || 0, height: height || 0, html: html || '', @@ -83,6 +105,7 @@ class LinkDetailsExtractor author_name: author_name || '', author_url: author_url || '', embed_url: embed_url || '', + language: language, } end @@ -90,6 +113,14 @@ class LinkDetailsExtractor player_url.present? ? :video : :link end + def link_type + if structured_data&.type == 'NewsArticle' || opengraph_tag('og:type') == 'article' + :article + else + :unknown + end + end + def html player_url.present? ? content_tag(:iframe, nil, src: player_url, width: width, height: height, allowtransparency: 'true', scrolling: 'no', frameborder: '0') : nil end @@ -138,6 +169,14 @@ class LinkDetailsExtractor valid_url_or_nil(opengraph_tag('twitter:player:stream')) end + def language + valid_locale_or_nil(structured_data&.language || opengraph_tag('og:locale') || document.xpath('//html').map { |element| element['lang'] }.first) + end + + def icon + valid_url_or_nil(structured_data&.publisher_icon || link_tag('apple-touch-icon') || link_tag('shortcut icon')) + end + private def player_url @@ -162,6 +201,14 @@ class LinkDetailsExtractor nil end + def valid_locale_or_nil(str) + return nil if str.blank? + + code, = str.split(/_-/) # Strip out the region from e.g. en_US or ja-JA + locale = ISO_639.find(code) + locale&.alpha2 + end + def link_tag(name) document.xpath("//link[@rel=\"#{name}\"]").map { |link| link['href'] }.first end diff --git a/app/mailers/admin_mailer.rb b/app/mailers/admin_mailer.rb index 11fd09e30..0fbd9932d 100644 --- a/app/mailers/admin_mailer.rb +++ b/app/mailers/admin_mailer.rb @@ -25,13 +25,25 @@ class AdminMailer < ApplicationMailer end end - def new_trending_tag(recipient, tag) - @tag = tag - @me = recipient - @instance = Rails.configuration.x.local_domain + def new_trending_tags(recipient, tags) + @tags = tags + @me = recipient + @instance = Rails.configuration.x.local_domain + @lowest_trending_tag = Trends.tags.get(true, Trends::Tags::REVIEW_THRESHOLD).last + + locale_for_account(@me) do + mail to: @me.user_email, subject: I18n.t('admin_mailer.new_trending_tags.subject', instance: @instance) + end + end + + def new_trending_links(recipient, links) + @links = links + @me = recipient + @instance = Rails.configuration.x.local_domain + @lowest_trending_link = Trends.links.get(true, Trends::Links::REVIEW_THRESHOLD).last locale_for_account(@me) do - mail to: @me.user_email, subject: I18n.t('admin_mailer.new_trending_tag.subject', instance: @instance, name: @tag.name) + mail to: @me.user_email, subject: I18n.t('admin_mailer.new_trending_links.subject', instance: @instance) end end end diff --git a/app/models/account_statuses_cleanup_policy.rb b/app/models/account_statuses_cleanup_policy.rb index 0a9551ec2..0f78c1a54 100644 --- a/app/models/account_statuses_cleanup_policy.rb +++ b/app/models/account_statuses_cleanup_policy.rb @@ -4,8 +4,8 @@ # # Table name: account_statuses_cleanup_policies # -# id :bigint not null, primary key -# account_id :bigint not null +# id :bigint(8) not null, primary key +# account_id :bigint(8) not null # enabled :boolean default(TRUE), not null # min_status_age :integer default(1209600), not null # keep_direct :boolean default(TRUE), not null diff --git a/app/models/form/preview_card_batch.rb b/app/models/form/preview_card_batch.rb new file mode 100644 index 000000000..5f6e6522a --- /dev/null +++ b/app/models/form/preview_card_batch.rb @@ -0,0 +1,65 @@ +# frozen_string_literal: true + +class Form::PreviewCardBatch + include ActiveModel::Model + include Authorization + + attr_accessor :preview_card_ids, :action, :current_account, :precision + + def save + case action + when 'approve' + approve! + when 'approve_all' + approve_all! + when 'reject' + reject! + when 'reject_all' + reject_all! + end + end + + private + + def preview_cards + @preview_cards ||= PreviewCard.where(id: preview_card_ids) + end + + def preview_card_providers + @preview_card_providers ||= preview_cards.map(&:domain).uniq.map { |domain| PreviewCardProvider.matching_domain(domain) || PreviewCardProvider.new(domain: domain) } + end + + def approve! + preview_cards.each { |preview_card| authorize(preview_card, :update?) } + preview_cards.update_all(trendable: true) + end + + def approve_all! + preview_card_providers.each do |provider| + authorize(provider, :update?) + provider.update(trendable: true, reviewed_at: action_time) + end + + # Reset any individual overrides + preview_cards.update_all(trendable: nil) + end + + def reject! + preview_cards.each { |preview_card| authorize(preview_card, :update?) } + preview_cards.update_all(trendable: false) + end + + def reject_all! + preview_card_providers.each do |provider| + authorize(provider, :update?) + provider.update(trendable: false, reviewed_at: action_time) + end + + # Reset any individual overrides + preview_cards.update_all(trendable: nil) + end + + def action_time + @action_time ||= Time.now.utc + end +end diff --git a/app/models/form/preview_card_provider_batch.rb b/app/models/form/preview_card_provider_batch.rb new file mode 100644 index 000000000..e6ab3d8fa --- /dev/null +++ b/app/models/form/preview_card_provider_batch.rb @@ -0,0 +1,33 @@ +# frozen_string_literal: true + +class Form::PreviewCardProviderBatch + include ActiveModel::Model + include Authorization + + attr_accessor :preview_card_provider_ids, :action, :current_account + + def save + case action + when 'approve' + approve! + when 'reject' + reject! + end + end + + private + + def preview_card_providers + PreviewCardProvider.where(id: preview_card_provider_ids) + end + + def approve! + preview_card_providers.each { |provider| authorize(provider, :update?) } + preview_card_providers.update_all(trendable: true, reviewed_at: Time.now.utc) + end + + def reject! + preview_card_providers.each { |provider| authorize(provider, :update?) } + preview_card_providers.update_all(trendable: false, reviewed_at: Time.now.utc) + end +end diff --git a/app/models/form/tag_batch.rb b/app/models/form/tag_batch.rb index fd517a1a6..b9330745f 100644 --- a/app/models/form/tag_batch.rb +++ b/app/models/form/tag_batch.rb @@ -23,11 +23,15 @@ class Form::TagBatch def approve! tags.each { |tag| authorize(tag, :update?) } - tags.update_all(trendable: true, reviewed_at: Time.now.utc) + tags.update_all(trendable: true, reviewed_at: action_time) end def reject! tags.each { |tag| authorize(tag, :update?) } - tags.update_all(trendable: false, reviewed_at: Time.now.utc) + tags.update_all(trendable: false, reviewed_at: action_time) + end + + def action_time + @action_time ||= Time.now.utc end end diff --git a/app/models/preview_card.rb b/app/models/preview_card.rb index bca3a3ce8..f2ab8ecab 100644 --- a/app/models/preview_card.rb +++ b/app/models/preview_card.rb @@ -24,6 +24,11 @@ # embed_url :string default(""), not null # image_storage_schema_version :integer # blurhash :string +# language :string +# max_score :float +# max_score_at :datetime +# trendable :boolean +# link_type :integer # class PreviewCard < ApplicationRecord @@ -40,6 +45,7 @@ class PreviewCard < ApplicationRecord self.inheritance_column = false enum type: [:link, :photo, :video, :rich] + enum link_type: [:unknown, :article] has_and_belongs_to_many :statuses @@ -54,6 +60,32 @@ class PreviewCard < ApplicationRecord before_save :extract_dimensions, if: :link? + def appropriate_for_trends? + link? && article? && title.present? && description.present? && image.present? && provider_name.present? + end + + def domain + @domain ||= Addressable::URI.parse(url).normalized_host + end + + def provider + @provider ||= PreviewCardProvider.matching_domain(domain) + end + + def trendable? + if attributes['trendable'].nil? + provider&.trendable? + else + attributes['trendable'] + end + end + + def requires_review_notification? + attributes['trendable'].nil? && (provider.nil? || provider.requires_review_notification?) + end + + attr_writer :provider + def local? false end @@ -69,11 +101,14 @@ class PreviewCard < ApplicationRecord save! end + def history + @history ||= Trends::History.new('links', id) + end + class << self private - # rubocop:disable Naming/MethodParameterName - def image_styles(f) + def image_styles(file) styles = { original: { geometry: '400x400>', @@ -83,10 +118,9 @@ class PreviewCard < ApplicationRecord }, } - styles[:original][:format] = 'jpg' if f.instance.image_content_type == 'image/gif' + styles[:original][:format] = 'jpg' if file.instance.image_content_type == 'image/gif' styles end - # rubocop:enable Naming/MethodParameterName end private diff --git a/app/models/preview_card_filter.rb b/app/models/preview_card_filter.rb new file mode 100644 index 000000000..8dda9989c --- /dev/null +++ b/app/models/preview_card_filter.rb @@ -0,0 +1,53 @@ +# frozen_string_literal: true + +class PreviewCardFilter + KEYS = %i( + trending + ).freeze + + attr_reader :params + + def initialize(params) + @params = params + end + + def results + scope = PreviewCard.unscoped + + params.each do |key, value| + next if key.to_s == 'page' + + scope.merge!(scope_for(key, value.to_s.strip)) if value.present? + end + + scope + end + + private + + def scope_for(key, value) + case key.to_s + when 'trending' + trending_scope(value) + else + raise "Unknown filter: #{key}" + end + end + + def trending_scope(value) + ids = begin + case value.to_s + when 'allowed' + Trends.links.currently_trending_ids(true, -1) + else + Trends.links.currently_trending_ids(false, -1) + end + end + + if ids.empty? + PreviewCard.none + else + PreviewCard.joins("join unnest(array[#{ids.map(&:to_i).join(',')}]::integer[]) with ordinality as x (id, ordering) on preview_cards.id = x.id").order('x.ordering') + end + end +end diff --git a/app/models/preview_card_provider.rb b/app/models/preview_card_provider.rb new file mode 100644 index 000000000..15b24e2bd --- /dev/null +++ b/app/models/preview_card_provider.rb @@ -0,0 +1,57 @@ +# frozen_string_literal: true +# == Schema Information +# +# Table name: preview_card_providers +# +# id :bigint(8) not null, primary key +# domain :string default(""), not null +# icon_file_name :string +# icon_content_type :string +# icon_file_size :bigint(8) +# icon_updated_at :datetime +# trendable :boolean +# reviewed_at :datetime +# requested_review_at :datetime +# created_at :datetime not null +# updated_at :datetime not null +# + +class PreviewCardProvider < ApplicationRecord + include DomainNormalizable + include Attachmentable + + ICON_MIME_TYPES = %w(image/x-icon image/vnd.microsoft.icon image/png).freeze + LIMIT = 1.megabyte + + validates :domain, presence: true, uniqueness: true, domain: true + + has_attached_file :icon, styles: { static: { format: 'png', convert_options: '-coalesce -strip' } }, validate_media_type: false + validates_attachment :icon, content_type: { content_type: ICON_MIME_TYPES }, size: { less_than: LIMIT } + remotable_attachment :icon, LIMIT + + scope :trendable, -> { where(trendable: true) } + scope :not_trendable, -> { where(trendable: false) } + scope :reviewed, -> { where.not(reviewed_at: nil) } + scope :pending_review, -> { where(reviewed_at: nil) } + + def requires_review? + reviewed_at.nil? + end + + def reviewed? + reviewed_at.present? + end + + def requested_review? + requested_review_at.present? + end + + def requires_review_notification? + requires_review? && !requested_review? + end + + def self.matching_domain(domain) + segments = domain.split('.') + where(domain: segments.map.with_index { |_, i| segments[i..-1].join('.') }).order(Arel.sql('char_length(domain) desc')).first + end +end diff --git a/app/models/preview_card_provider_filter.rb b/app/models/preview_card_provider_filter.rb new file mode 100644 index 000000000..1e90d3c9d --- /dev/null +++ b/app/models/preview_card_provider_filter.rb @@ -0,0 +1,49 @@ +# frozen_string_literal: true + +class PreviewCardProviderFilter + KEYS = %i( + status + ).freeze + + attr_reader :params + + def initialize(params) + @params = params + end + + def results + scope = PreviewCardProvider.unscoped + + params.each do |key, value| + next if key.to_s == 'page' + + scope.merge!(scope_for(key, value.to_s.strip)) if value.present? + end + + scope.order(domain: :asc) + end + + private + + def scope_for(key, value) + case key.to_s + when 'status' + status_scope(value) + else + raise "Unknown filter: #{key}" + end + end + + def status_scope(value) + case value.to_s + when 'approved' + PreviewCardProvider.trendable + when 'rejected' + PreviewCardProvider.not_trendable + when 'pending_review' + PreviewCardProvider.pending_review + else + raise "Unknown status: #{value}" + end + end +end diff --git a/app/models/tag.rb b/app/models/tag.rb index dcce28391..f35d92b5d 100644 --- a/app/models/tag.rb +++ b/app/models/tag.rb @@ -36,6 +36,7 @@ class Tag < ApplicationRecord scope :usable, -> { where(usable: [true, nil]) } scope :listable, -> { where(listable: [true, nil]) } scope :trendable, -> { Setting.trendable_by_default ? where(trendable: [true, nil]) : where(trendable: true) } + scope :not_trendable, -> { where(trendable: false) } scope :recently_used, ->(account) { joins(:statuses).where(statuses: { id: account.statuses.select(:id).limit(1000) }).group(:id).order(Arel.sql('count(*) desc')) } scope :matches_name, ->(term) { where(arel_table[:name].lower.matches(arel_table.lower("#{sanitize_sql_like(Tag.normalize(term))}%"), nil, true)) } # Search with case-sensitive to use B-tree index @@ -75,28 +76,12 @@ class Tag < ApplicationRecord requested_review_at.present? end - def use!(account, status: nil, at_time: Time.now.utc) - TrendingTags.record_use!(self, account, status: status, at_time: at_time) - end - - def trending? - TrendingTags.trending?(self) + def requires_review_notification? + requires_review? && !requested_review? end def history - days = [] - - 7.times do |i| - day = i.days.ago.beginning_of_day.to_i - - days << { - day: day.to_s, - uses: Redis.current.get("activity:tags:#{id}:#{day}") || '0', - accounts: Redis.current.pfcount("activity:tags:#{id}:#{day}:accounts").to_s, - } - end - - days + @history ||= Trends::History.new('tags', id) end class << self diff --git a/app/models/tag_filter.rb b/app/models/tag_filter.rb index 85bfcbea5..ecdb52503 100644 --- a/app/models/tag_filter.rb +++ b/app/models/tag_filter.rb @@ -2,13 +2,8 @@ class TagFilter KEYS = %i( - directory - reviewed - unreviewed - pending_review - popular - active - name + trending + status ).freeze attr_reader :params @@ -18,7 +13,13 @@ class TagFilter end def results - scope = Tag.unscoped + scope = begin + if params[:status] == 'pending_review' + Tag.unscoped + else + trending_scope + end + end params.each do |key, value| next if key.to_s == 'page' @@ -26,27 +27,40 @@ class TagFilter scope.merge!(scope_for(key, value.to_s.strip)) if value.present? end - scope.order(id: :desc) + scope end private def scope_for(key, value) case key.to_s - when 'reviewed' - Tag.reviewed.order(reviewed_at: :desc) - when 'unreviewed' - Tag.unreviewed - when 'pending_review' - Tag.pending_review.order(requested_review_at: :desc) - when 'popular' - Tag.order('max_score DESC NULLS LAST') - when 'active' - Tag.order('last_status_at DESC NULLS LAST') - when 'name' - Tag.matches_name(value) + when 'status' + status_scope(value) else raise "Unknown filter: #{key}" end end + + def trending_scope + ids = Trends.tags.currently_trending_ids(false, -1) + + if ids.empty? + Tag.none + else + Tag.joins("join unnest(array[#{ids.map(&:to_i).join(',')}]::integer[]) with ordinality as x (id, ordering) on tags.id = x.id").order('x.ordering') + end + end + + def status_scope(value) + case value.to_s + when 'approved' + Tag.trendable + when 'rejected' + Tag.not_trendable + when 'pending_review' + Tag.pending_review + else + raise "Unknown status: #{value}" + end + end end diff --git a/app/models/trending_tags.rb b/app/models/trending_tags.rb deleted file mode 100644 index 31890b082..000000000 --- a/app/models/trending_tags.rb +++ /dev/null @@ -1,128 +0,0 @@ -# frozen_string_literal: true - -class TrendingTags - KEY = 'trending_tags' - EXPIRE_HISTORY_AFTER = 7.days.seconds - EXPIRE_TRENDS_AFTER = 1.day.seconds - THRESHOLD = 5 - LIMIT = 10 - REVIEW_THRESHOLD = 3 - MAX_SCORE_COOLDOWN = 2.days.freeze - MAX_SCORE_HALFLIFE = 2.hours.freeze - - class << self - include Redisable - - def record_use!(tag, account, status: nil, at_time: Time.now.utc) - return unless tag.usable? && !account.silenced? - - # Even if a tag is not allowed to trend, we still need to - # record the stats since they can be displayed in other places - increment_historical_use!(tag.id, at_time) - increment_unique_use!(tag.id, account.id, at_time) - increment_use!(tag.id, at_time) - - # Only update when the tag was last used once every 12 hours - # and only if a status is given (lets use ignore reblogs) - tag.update(last_status_at: at_time) if status.present? && (tag.last_status_at.nil? || (tag.last_status_at < at_time && tag.last_status_at < 12.hours.ago)) - end - - def update!(at_time = Time.now.utc) - tag_ids = redis.smembers("#{KEY}:used:#{at_time.beginning_of_day.to_i}") + redis.zrange(KEY, 0, -1) - tags = Tag.trendable.where(id: tag_ids.uniq) - - # First pass to calculate scores and update the set - - tags.each do |tag| - expected = redis.pfcount("activity:tags:#{tag.id}:#{(at_time - 1.day).beginning_of_day.to_i}:accounts").to_f - expected = 1.0 if expected.zero? - observed = redis.pfcount("activity:tags:#{tag.id}:#{at_time.beginning_of_day.to_i}:accounts").to_f - max_time = tag.max_score_at - max_score = tag.max_score - max_score = 0 if max_time.nil? || max_time < (at_time - MAX_SCORE_COOLDOWN) - - score = begin - if expected > observed || observed < THRESHOLD - 0 - else - ((observed - expected)**2) / expected - end - end - - if score > max_score - max_score = score - max_time = at_time - - # Not interested in triggering any callbacks for this - tag.update_columns(max_score: max_score, max_score_at: max_time) - end - - decaying_score = max_score * (0.5**((at_time.to_f - max_time.to_f) / MAX_SCORE_HALFLIFE.to_f)) - - if decaying_score.zero? - redis.zrem(KEY, tag.id) - else - redis.zadd(KEY, decaying_score, tag.id) - end - end - - users_for_review = User.staff.includes(:account).to_a.select(&:allows_trending_tag_emails?) - - # Second pass to notify about previously unreviewed trends - - tags.each do |tag| - current_rank = redis.zrevrank(KEY, tag.id) - needs_review_notification = tag.requires_review? && !tag.requested_review? - rank_passes_threshold = current_rank.present? && current_rank <= REVIEW_THRESHOLD - - next unless !tag.trendable? && rank_passes_threshold && needs_review_notification - - tag.touch(:requested_review_at) - - users_for_review.each do |user| - AdminMailer.new_trending_tag(user.account, tag).deliver_later! - end - end - - # Trim older items - - redis.zremrangebyrank(KEY, 0, -(LIMIT + 1)) - redis.zremrangebyscore(KEY, '(0.3', '-inf') - end - - def get(limit, filtered: true) - tag_ids = redis.zrevrange(KEY, 0, LIMIT - 1).map(&:to_i) - - tags = Tag.where(id: tag_ids) - tags = tags.trendable if filtered - tags = tags.index_by(&:id) - - tag_ids.map { |tag_id| tags[tag_id] }.compact.take(limit) - end - - def trending?(tag) - rank = redis.zrevrank(KEY, tag.id) - rank.present? && rank < LIMIT - end - - private - - def increment_historical_use!(tag_id, at_time) - key = "activity:tags:#{tag_id}:#{at_time.beginning_of_day.to_i}" - redis.incrby(key, 1) - redis.expire(key, EXPIRE_HISTORY_AFTER) - end - - def increment_unique_use!(tag_id, account_id, at_time) - key = "activity:tags:#{tag_id}:#{at_time.beginning_of_day.to_i}:accounts" - redis.pfadd(key, account_id) - redis.expire(key, EXPIRE_HISTORY_AFTER) - end - - def increment_use!(tag_id, at_time) - key = "#{KEY}:used:#{at_time.beginning_of_day.to_i}" - redis.sadd(key, tag_id) - redis.expire(key, EXPIRE_HISTORY_AFTER) - end - end -end diff --git a/app/models/trends.rb b/app/models/trends.rb new file mode 100644 index 000000000..7dd3a9c87 --- /dev/null +++ b/app/models/trends.rb @@ -0,0 +1,27 @@ +# frozen_string_literal: true + +module Trends + def self.table_name_prefix + 'trends_' + end + + def self.links + @links ||= Trends::Links.new + end + + def self.tags + @tags ||= Trends::Tags.new + end + + def self.refresh! + [links, tags].each(&:refresh) + end + + def self.request_review! + [links, tags].each(&:request_review) if enabled? + end + + def self.enabled? + Setting.trends + end +end diff --git a/app/models/trends/base.rb b/app/models/trends/base.rb new file mode 100644 index 000000000..b767dcb1a --- /dev/null +++ b/app/models/trends/base.rb @@ -0,0 +1,80 @@ +# frozen_string_literal: true + +class Trends::Base + include Redisable + + class_attribute :default_options + + attr_reader :options + + # @param [Hash] options + # @option options [Integer] :threshold Minimum amount of uses by unique accounts to begin calculating the score + # @option options [Integer] :review_threshold Minimum rank (lower = better) before requesting a review + # @option options [ActiveSupport::Duration] :max_score_cooldown For this amount of time, the peak score (if bigger than current score) is decayed-from + # @option options [ActiveSupport::Duration] :max_score_halflife How quickly a peak score decays + def initialize(options = {}) + @options = self.class.default_options.merge(options) + end + + def register(_status) + raise NotImplementedError + end + + def add(*) + raise NotImplementedError + end + + def refresh(*) + raise NotImplementedError + end + + def request_review + raise NotImplementedError + end + + def get(*) + raise NotImplementedError + end + + def score(id) + redis.zscore("#{key_prefix}:all", id) || 0 + end + + def rank(id) + redis.zrevrank("#{key_prefix}:allowed", id) + end + + def currently_trending_ids(allowed, limit) + redis.zrevrange(allowed ? "#{key_prefix}:allowed" : "#{key_prefix}:all", 0, limit.positive? ? limit - 1 : limit).map(&:to_i) + end + + protected + + def key_prefix + raise NotImplementedError + end + + def recently_used_ids(at_time = Time.now.utc) + redis.smembers(used_key(at_time)).map(&:to_i) + end + + def record_used_id(id, at_time = Time.now.utc) + redis.sadd(used_key(at_time), id) + redis.expire(used_key(at_time), 1.day.seconds) + end + + def trim_older_items + redis.zremrangebyscore("#{key_prefix}:all", '-inf', '(1') + redis.zremrangebyscore("#{key_prefix}:allowed", '-inf', '(1') + end + + def score_at_rank(rank) + redis.zrevrange("#{key_prefix}:allowed", 0, rank, with_scores: true).last&.last || 0 + end + + private + + def used_key(at_time) + "#{key_prefix}:used:#{at_time.beginning_of_day.to_i}" + end +end diff --git a/app/models/trends/history.rb b/app/models/trends/history.rb new file mode 100644 index 000000000..608e33792 --- /dev/null +++ b/app/models/trends/history.rb @@ -0,0 +1,98 @@ +# frozen_string_literal: true + +class Trends::History + include Enumerable + + class Aggregate + include Redisable + + def initialize(prefix, id, date_range) + @days = date_range.map { |date| Day.new(prefix, id, date.to_time(:utc)) } + end + + def uses + redis.mget(*@days.map { |day| day.key_for(:uses) }).map(&:to_i).sum + end + + def accounts + redis.pfcount(*@days.map { |day| day.key_for(:accounts) }) + end + end + + class Day + include Redisable + + EXPIRE_AFTER = 14.days.seconds + + def initialize(prefix, id, day) + @prefix = prefix + @id = id + @day = day.beginning_of_day + end + + attr_reader :day + + def accounts + redis.pfcount(key_for(:accounts)) + end + + def uses + redis.get(key_for(:uses))&.to_i || 0 + end + + def add(account_id) + redis.pipelined do + redis.incrby(key_for(:uses), 1) + redis.pfadd(key_for(:accounts), account_id) + redis.expire(key_for(:uses), EXPIRE_AFTER) + redis.expire(key_for(:accounts), EXPIRE_AFTER) + end + end + + def as_json + { day: day.to_i.to_s, accounts: accounts.to_s, uses: uses.to_s } + end + + def key_for(suffix) + case suffix + when :accounts + "#{key_prefix}:#{suffix}" + when :uses + key_prefix + end + end + + def key_prefix + "activity:#{@prefix}:#{@id}:#{day.to_i}" + end + end + + def initialize(prefix, id) + @prefix = prefix + @id = id + end + + def get(date) + Day.new(@prefix, @id, date) + end + + def add(account_id, at_time = Time.now.utc) + Day.new(@prefix, @id, at_time).add(account_id) + end + + def aggregate(date_range) + Aggregate.new(@prefix, @id, date_range) + end + + def each(&block) + if block_given? + (0...7).map { |i| block.call(get(i.days.ago)) } + else + to_enum(:each) + end + end + + def as_json(*) + map(&:as_json) + end +end diff --git a/app/models/trends/links.rb b/app/models/trends/links.rb new file mode 100644 index 000000000..a0d65138b --- /dev/null +++ b/app/models/trends/links.rb @@ -0,0 +1,117 @@ +# frozen_string_literal: true + +class Trends::Links < Trends::Base + PREFIX = 'trending_links' + + self.default_options = { + threshold: 15, + review_threshold: 10, + max_score_cooldown: 2.days.freeze, + max_score_halflife: 8.hours.freeze, + } + + def register(status, at_time = Time.now.utc) + original_status = status.reblog? ? status.reblog : status + + return unless original_status.public_visibility? && status.public_visibility? && + !original_status.account.silenced? && !status.account.silenced? && + !original_status.spoiler_text? + + original_status.preview_cards.each do |preview_card| + add(preview_card, status.account_id, at_time) if preview_card.appropriate_for_trends? + end + end + + def add(preview_card, account_id, at_time = Time.now.utc) + preview_card.history.add(account_id, at_time) + record_used_id(preview_card.id, at_time) + end + + def get(allowed, limit) + preview_card_ids = currently_trending_ids(allowed, limit) + preview_cards = PreviewCard.where(id: preview_card_ids).index_by(&:id) + preview_card_ids.map { |id| preview_cards[id] }.compact + end + + def refresh(at_time = Time.now.utc) + preview_cards = PreviewCard.where(id: (recently_used_ids(at_time) + currently_trending_ids(false, -1)).uniq) + calculate_scores(preview_cards, at_time) + trim_older_items + end + + def request_review + preview_cards = PreviewCard.where(id: currently_trending_ids(false, -1)) + + preview_cards_requiring_review = preview_cards.filter_map do |preview_card| + next unless would_be_trending?(preview_card.id) && !preview_card.trendable? && preview_card.requires_review_notification? + + if preview_card.provider.nil? + preview_card.provider = PreviewCardProvider.create(domain: preview_card.domain, requested_review_at: Time.now.utc) + else + preview_card.provider.touch(:requested_review_at) + end + + preview_card + end + + return if preview_cards_requiring_review.empty? + + User.staff.includes(:account).find_each do |user| + AdminMailer.new_trending_links(user.account, preview_cards_requiring_review).deliver_later! if user.allows_trending_tag_emails? + end + end + + protected + + def key_prefix + PREFIX + end + + private + + def calculate_scores(preview_cards, at_time) + preview_cards.each do |preview_card| + expected = preview_card.history.get(at_time - 1.day).accounts.to_f + expected = 1.0 if expected.zero? + observed = preview_card.history.get(at_time).accounts.to_f + max_time = preview_card.max_score_at + max_score = preview_card.max_score + max_score = 0 if max_time.nil? || max_time < (at_time - options[:max_score_cooldown]) + + score = begin + if expected > observed || observed < options[:threshold] + 0 + else + ((observed - expected)**2) / expected + end + end + + if score > max_score + max_score = score + max_time = at_time + + # Not interested in triggering any callbacks for this + preview_card.update_columns(max_score: max_score, max_score_at: max_time) + end + + decaying_score = max_score * (0.5**((at_time.to_f - max_time.to_f) / options[:max_score_halflife].to_f)) + + if decaying_score.zero? + redis.zrem("#{PREFIX}:all", preview_card.id) + redis.zrem("#{PREFIX}:allowed", preview_card.id) + else + redis.zadd("#{PREFIX}:all", decaying_score, preview_card.id) + + if preview_card.trendable? + redis.zadd("#{PREFIX}:allowed", decaying_score, preview_card.id) + else + redis.zrem("#{PREFIX}:allowed", preview_card.id) + end + end + end + end + + def would_be_trending?(id) + score(id) > score_at_rank(options[:review_threshold] - 1) + end +end diff --git a/app/models/trends/tags.rb b/app/models/trends/tags.rb new file mode 100644 index 000000000..13e0ab56b --- /dev/null +++ b/app/models/trends/tags.rb @@ -0,0 +1,111 @@ +# frozen_string_literal: true + +class Trends::Tags < Trends::Base + PREFIX = 'trending_tags' + + self.default_options = { + threshold: 15, + review_threshold: 10, + max_score_cooldown: 2.days.freeze, + max_score_halflife: 4.hours.freeze, + } + + def register(status, at_time = Time.now.utc) + original_status = status.reblog? ? status.reblog : status + + return unless original_status.public_visibility? && status.public_visibility? && + !original_status.account.silenced? && !status.account.silenced? + + original_status.tags.each do |tag| + add(tag, status.account_id, at_time) if tag.usable? + end + end + + def add(tag, account_id, at_time = Time.now.utc) + tag.history.add(account_id, at_time) + record_used_id(tag.id, at_time) + end + + def refresh(at_time = Time.now.utc) + tags = Tag.where(id: (recently_used_ids(at_time) + currently_trending_ids(false, -1)).uniq) + calculate_scores(tags, at_time) + trim_older_items + end + + def get(allowed, limit) + tag_ids = currently_trending_ids(allowed, limit) + tags = Tag.where(id: tag_ids).index_by(&:id) + tag_ids.map { |id| tags[id] }.compact + end + + def request_review + tags = Tag.where(id: currently_trending_ids(false, -1)) + + tags_requiring_review = tags.filter_map do |tag| + next unless would_be_trending?(tag.id) && !tag.trendable? && tag.requires_review_notification? + + tag.touch(:requested_review_at) + tag + end + + return if tags_requiring_review.empty? + + User.staff.includes(:account).find_each do |user| + AdminMailer.new_trending_tags(user.account, tags_requiring_review).deliver_later! if user.allows_trending_tag_emails? + end + end + + protected + + def key_prefix + PREFIX + end + + private + + def calculate_scores(tags, at_time) + tags.each do |tag| + expected = tag.history.get(at_time - 1.day).accounts.to_f + expected = 1.0 if expected.zero? + observed = tag.history.get(at_time).accounts.to_f + max_time = tag.max_score_at + max_score = tag.max_score + max_score = 0 if max_time.nil? || max_time < (at_time - options[:max_score_cooldown]) + + score = begin + if expected > observed || observed < options[:threshold] + 0 + else + ((observed - expected)**2) / expected + end + end + + if score > max_score + max_score = score + max_time = at_time + + # Not interested in triggering any callbacks for this + tag.update_columns(max_score: max_score, max_score_at: max_time) + end + + decaying_score = max_score * (0.5**((at_time.to_f - max_time.to_f) / options[:max_score_halflife].to_f)) + + if decaying_score.zero? + redis.zrem("#{PREFIX}:all", tag.id) + redis.zrem("#{PREFIX}:allowed", tag.id) + else + redis.zadd("#{PREFIX}:all", decaying_score, tag.id) + + if tag.trendable? + redis.zadd("#{PREFIX}:allowed", decaying_score, tag.id) + else + redis.zrem("#{PREFIX}:allowed", tag.id) + end + end + end + end + + def would_be_trending?(id) + score(id) > score_at_rank(options[:review_threshold] - 1) + end +end diff --git a/app/policies/preview_card_policy.rb b/app/policies/preview_card_policy.rb new file mode 100644 index 000000000..4f485d7fc --- /dev/null +++ b/app/policies/preview_card_policy.rb @@ -0,0 +1,11 @@ +# frozen_string_literal: true + +class PreviewCardPolicy < ApplicationPolicy + def index? + staff? + end + + def update? + staff? + end +end diff --git a/app/policies/preview_card_provider_policy.rb b/app/policies/preview_card_provider_policy.rb new file mode 100644 index 000000000..598d54a5e --- /dev/null +++ b/app/policies/preview_card_provider_policy.rb @@ -0,0 +1,11 @@ +# frozen_string_literal: true + +class PreviewCardProviderPolicy < ApplicationPolicy + def index? + staff? + end + + def update? + staff? + end +end diff --git a/app/serializers/rest/trends/link_serializer.rb b/app/serializers/rest/trends/link_serializer.rb new file mode 100644 index 000000000..232483490 --- /dev/null +++ b/app/serializers/rest/trends/link_serializer.rb @@ -0,0 +1,5 @@ +# frozen_string_literal: true + +class REST::Trends::LinkSerializer < REST::PreviewCardSerializer + attributes :history +end diff --git a/app/services/fetch_link_card_service.rb b/app/services/fetch_link_card_service.rb index 51956ce7e..94dc6389f 100644 --- a/app/services/fetch_link_card_service.rb +++ b/app/services/fetch_link_card_service.rb @@ -50,7 +50,7 @@ class FetchLinkCardService < BaseService # We follow redirects, and ideally we want to save the preview card for # the destination URL and not any link shortener in-between, so here # we set the URL to the one of the last response in the redirect chain - @url = res.request.uri.to_s.to_s + @url = res.request.uri.to_s @card = PreviewCard.find_or_initialize_by(url: @url) if @card.url != @url if res.code == 200 && res.mime_type == 'text/html' @@ -66,6 +66,7 @@ class FetchLinkCardService < BaseService def attach_card @status.preview_cards << @card Rails.cache.delete(@status) + Trends.links.register(@status) end def parse_urls diff --git a/app/services/post_status_service.rb b/app/services/post_status_service.rb index 85aaec4d6..294ae43eb 100644 --- a/app/services/post_status_service.rb +++ b/app/services/post_status_service.rb @@ -91,7 +91,8 @@ class PostStatusService < BaseService end def postprocess_status! - LinkCrawlWorker.perform_async(@status.id) unless @status.spoiler_text? + Trends.tags.register(@status) + LinkCrawlWorker.perform_async(@status.id) DistributionWorker.perform_async(@status.id) ActivityPub::DistributionWorker.perform_async(@status.id) PollExpirationNotifyWorker.perform_at(@status.poll.expires_at, @status.poll.id) if @status.poll diff --git a/app/services/process_hashtags_service.rb b/app/services/process_hashtags_service.rb index c42b79db8..47277c56c 100644 --- a/app/services/process_hashtags_service.rb +++ b/app/services/process_hashtags_service.rb @@ -8,7 +8,7 @@ class ProcessHashtagsService < BaseService Tag.find_or_create_by_names(tags) do |tag| status.tags << tag records << tag - tag.use!(status.account, status: status, at_time: status.created_at) if status.public_visibility? + tag.update(last_status_at: status.created_at) if tag.last_status_at.nil? || (tag.last_status_at < status.created_at && tag.last_status_at < 12.hours.ago) end return unless status.distributable? diff --git a/app/services/reblog_service.rb b/app/services/reblog_service.rb index 744bdf567..ece91847a 100644 --- a/app/services/reblog_service.rb +++ b/app/services/reblog_service.rb @@ -30,12 +30,13 @@ class ReblogService < BaseService reblog = account.statuses.create!(reblog: reblogged_status, text: '', visibility: visibility, rate_limit: options[:with_rate_limit]) + Trends.tags.register(reblog) + Trends.links.register(reblog) DistributionWorker.perform_async(reblog.id) ActivityPub::DistributionWorker.perform_async(reblog.id) create_notification(reblog) bump_potential_friendship(account, reblog) - record_use(account, reblog) reblog end @@ -60,16 +61,6 @@ class ReblogService < BaseService PotentialFriendshipTracker.record(account.id, reblog.reblog.account_id, :reblog) end - def record_use(account, reblog) - return unless reblog.public_visibility? - - original_status = reblog.reblog - - original_status.tags.each do |tag| - tag.use!(account) - end - end - def build_json(reblog) Oj.dump(serialize_payload(ActivityPub::ActivityPresenter.from_status(reblog), ActivityPub::ActivitySerializer, signer: reblog.account)) end diff --git a/app/views/admin/dashboard/index.html.haml b/app/views/admin/dashboard/index.html.haml index 560eba7b4..895333a58 100644 --- a/app/views/admin/dashboard/index.html.haml +++ b/app/views/admin/dashboard/index.html.haml @@ -42,7 +42,7 @@ %span= t('admin.dashboard.pending_users_html', count: @pending_users_count) = fa_icon 'chevron-right fw' - = link_to admin_tags_path(pending_review: '1'), class: 'dashboard__quick-access' do + = link_to admin_trends_tags_path(status: 'pending_review'), class: 'dashboard__quick-access' do %span= t('admin.dashboard.pending_tags_html', count: @pending_tags_count) = fa_icon 'chevron-right fw' diff --git a/app/views/admin/tags/_tag.html.haml b/app/views/admin/tags/_tag.html.haml deleted file mode 100644 index ac0c72816..000000000 --- a/app/views/admin/tags/_tag.html.haml +++ /dev/null @@ -1,19 +0,0 @@ -.batch-table__row - - if batch_available - %label.batch-table__row__select.batch-table__row__select--aligned.batch-checkbox - = f.check_box :tag_ids, { multiple: true, include_hidden: false }, tag.id - - .directory__tag - = link_to admin_tag_path(tag.id) do - %h4 - = fa_icon 'hashtag' - = tag.name - - %small - = t('admin.tags.unique_uses_today', count: tag.history.first[:accounts]) - - - if tag.trending? - = fa_icon 'fire fw' - = t('admin.tags.trending_right_now') - - .trends__item__current= friendly_number_to_human tag.history.first[:uses] diff --git a/app/views/admin/tags/index.html.haml b/app/views/admin/tags/index.html.haml deleted file mode 100644 index d78f3c6d1..000000000 --- a/app/views/admin/tags/index.html.haml +++ /dev/null @@ -1,74 +0,0 @@ -- content_for :page_title do - = t('admin.tags.title') - -- content_for :header_tags do - = javascript_pack_tag 'admin', async: true, crossorigin: 'anonymous' - -.filters - .filter-subset - %strong= t('admin.tags.review') - %ul - %li= filter_link_to t('generic.all'), reviewed: nil, unreviewed: nil, pending_review: nil - %li= filter_link_to t('admin.tags.unreviewed'), unreviewed: '1', reviewed: nil, pending_review: nil - %li= filter_link_to t('admin.tags.reviewed'), reviewed: '1', unreviewed: nil, pending_review: nil - %li= filter_link_to safe_join([t('admin.accounts.moderation.pending'), "(#{Tag.pending_review.count})"], ' '), pending_review: '1', reviewed: nil, unreviewed: nil - - .filter-subset - %strong= t('generic.order_by') - %ul - %li= filter_link_to t('admin.tags.most_recent'), popular: nil, active: nil - %li= filter_link_to t('admin.tags.last_active'), active: '1', popular: nil - %li= filter_link_to t('admin.tags.most_popular'), popular: '1', active: nil - - -= form_tag admin_tags_url, method: 'GET', class: 'simple_form' do - .fields-group - - TagFilter::KEYS.each do |key| - = hidden_field_tag key, params[key] if params[key].present? - - - %i(name).each do |key| - .input.string.optional - = text_field_tag key, params[key], class: 'string optional', placeholder: I18n.t("admin.tags.#{key}") - - .actions - %button.button= t('admin.accounts.search') - = link_to t('admin.accounts.reset'), admin_tags_path, class: 'button negative' - -%hr.spacer/ - -= form_for(@form, url: batch_admin_tags_path) do |f| - = hidden_field_tag :page, params[:page] || 1 - - - TagFilter::KEYS.each do |key| - = hidden_field_tag key, params[key] if params[key].present? - - .batch-table.optional - .batch-table__toolbar - - if params[:pending_review] == '1' || params[:unreviewed] == '1' - %label.batch-table__toolbar__select.batch-checkbox-all - = check_box_tag :batch_checkbox_all, nil, false - .batch-table__toolbar__actions - = f.button safe_join([fa_icon('check'), t('admin.accounts.approve')]), name: :approve, class: 'table-action-link', type: :submit, data: { confirm: t('admin.reports.are_you_sure') } - - = f.button safe_join([fa_icon('times'), t('admin.accounts.reject')]), name: :reject, class: 'table-action-link', type: :submit, data: { confirm: t('admin.reports.are_you_sure') } - - else - .batch-table__toolbar__actions - %span.neutral-hint= t('generic.no_batch_actions_available') - - .batch-table__body - - if @tags.empty? - = nothing_here 'nothing-here--under-tabs' - - else - = render partial: 'tag', collection: @tags, locals: { f: f, batch_available: params[:pending_review] == '1' || params[:unreviewed] == '1' } - -= paginate @tags - -- if params[:pending_review] == '1' || params[:unreviewed] == '1' - %hr.spacer/ - - %div.action-buttons - %div - = link_to t('admin.accounts.approve_all'), approve_all_admin_tags_path, method: :post, data: { confirm: t('admin.accounts.are_you_sure') }, class: 'button' - - %div - = link_to t('admin.accounts.reject_all'), reject_all_admin_tags_path, method: :post, data: { confirm: t('admin.accounts.are_you_sure') }, class: 'button button--destructive' diff --git a/app/views/admin/tags/show.html.haml b/app/views/admin/tags/show.html.haml index c4caffda1..007dc005e 100644 --- a/app/views/admin/tags/show.html.haml +++ b/app/views/admin/tags/show.html.haml @@ -1,15 +1,50 @@ +- content_for :header_tags do + = javascript_pack_tag 'admin', async: true, crossorigin: 'anonymous' + - content_for :page_title do = "##{@tag.name}" -.dashboard__counters - %div - = link_to tag_url(@tag), target: '_blank', rel: 'noopener noreferrer' do - .dashboard__counters__num= number_with_delimiter @accounts_today - .dashboard__counters__label= t 'admin.tags.accounts_today' - %div - %div - .dashboard__counters__num= number_with_delimiter @accounts_week - .dashboard__counters__label= t 'admin.tags.accounts_week' +- content_for :heading_actions do + = l(@time_period.first) + = ' - ' + = l(@time_period.last) + +.dashboard + .dashboard__item + = react_admin_component :counter, measure: 'tag_accounts', start_at: @time_period.first, end_at: @time_period.last, params: { id: @tag.id }, label: t('admin.trends.tags.dashboard.tag_accounts_measure') + .dashboard__item + = react_admin_component :counter, measure: 'tag_uses', start_at: @time_period.first, end_at: @time_period.last, params: { id: @tag.id }, label: t('admin.trends.tags.dashboard.tag_uses_measure') + .dashboard__item + = react_admin_component :counter, measure: 'tag_servers', start_at: @time_period.first, end_at: @time_period.last, params: { id: @tag.id }, label: t('admin.trends.tags.dashboard.tag_servers_measure') + .dashboard__item + = react_admin_component :dimension, dimension: 'tag_servers', start_at: @time_period.first, end_at: @time_period.last, params: { id: @tag.id }, limit: 8, label: t('admin.trends.tags.dashboard.tag_servers_dimension') + .dashboard__item + = react_admin_component :dimension, dimension: 'tag_languages', start_at: @time_period.first, end_at: @time_period.last, params: { id: @tag.id }, limit: 8, label: t('admin.trends.tags.dashboard.tag_languages_dimension') + .dashboard__item + = link_to admin_tag_path(@tag.id), class: ['dashboard__quick-access', @tag.usable? ? 'positive' : 'negative'] do + - if @tag.usable? + %span= t('admin.trends.tags.usable') + = fa_icon 'check fw' + - else + %span= t('admin.trends.tags.not_usable') + = fa_icon 'lock fw' + + = link_to admin_tag_path(@tag.id), class: ['dashboard__quick-access', @tag.trendable? ? 'positive' : 'negative'] do + - if @tag.trendable? + %span= t('admin.trends.tags.trendable') + = fa_icon 'check fw' + - else + %span= t('admin.trends.tags.not_trendable') + = fa_icon 'lock fw' + + + = link_to admin_tag_path(@tag.id), class: ['dashboard__quick-access', @tag.listable? ? 'positive' : 'negative'] do + - if @tag.listable? + %span= t('admin.trends.tags.listable') + = fa_icon 'check fw' + - else + %span= t('admin.trends.tags.not_listable') + = fa_icon 'lock fw' %hr.spacer/ @@ -26,18 +61,3 @@ .actions = f.button :button, t('generic.save_changes'), type: :submit - -%hr.spacer/ - -%h3= t 'admin.tags.breakdown' - -.table-wrapper - %table.table - %tbody - - total = @usage_by_domain.sum(&:last).to_f - - - @usage_by_domain.each do |(domain, count)| - %tr - %th= domain || site_hostname - %td= number_to_percentage((count / total) * 100, precision: 1) - %td= number_with_delimiter count diff --git a/app/views/admin/trends/links/_preview_card.html.haml b/app/views/admin/trends/links/_preview_card.html.haml new file mode 100644 index 000000000..dfed13b68 --- /dev/null +++ b/app/views/admin/trends/links/_preview_card.html.haml @@ -0,0 +1,30 @@ +.batch-table__row{ class: [preview_card.provider&.requires_review? && 'batch-table__row--attention', !preview_card.provider&.requires_review? && !preview_card.trendable? && 'batch-table__row--muted'] } + %label.batch-table__row__select.batch-table__row__select--aligned.batch-checkbox + = f.check_box :preview_card_ids, { multiple: true, include_hidden: false }, preview_card.id + + .batch-table__row__content.pending-account + .pending-account__header + = link_to preview_card.title, preview_card.url + + %br/ + + - if preview_card.provider_name.present? + = preview_card.provider_name + • + + - if preview_card.language.present? + = human_locale(preview_card.language) + • + + = t('admin.trends.links.shared_by_over_week', count: preview_card.history.reduce(0) { |sum, day| sum + day.accounts }) + + - if preview_card.trendable? && (rank = Trends.links.rank(preview_card.id)) + • + %abbr{ title: t('admin.trends.tags.current_score', score: Trends.links.score(preview_card.id)) }= t('admin.trends.tags.trending_rank', rank: rank + 1) + + - if preview_card.max_score_at && preview_card.max_score_at >= Trends::Links::MAX_SCORE_COOLDOWN.ago && preview_card.max_score_at < 1.day.ago + • + = t('admin.trends.tags.peaked_on_and_decaying', date: l(preview_card.max_score_at.to_date, format: :short)) + - elsif preview_card.provider&.requires_review? + • + = t('admin.trends.pending_review') diff --git a/app/views/admin/trends/links/index.html.haml b/app/views/admin/trends/links/index.html.haml new file mode 100644 index 000000000..240ae722b --- /dev/null +++ b/app/views/admin/trends/links/index.html.haml @@ -0,0 +1,41 @@ +- content_for :page_title do + = t('admin.trends.links.title') + +- content_for :header_tags do + = javascript_pack_tag 'admin', async: true, crossorigin: 'anonymous' + +.filters + .filter-subset + %strong= t('admin.trends.trending') + %ul + %li= filter_link_to t('generic.all'), trending: nil + %li= filter_link_to t('admin.trends.only_allowed'), trending: 'allowed' + .back-link + = link_to admin_trends_links_preview_card_providers_path do + = t('admin.trends.preview_card_providers.title') + = fa_icon 'chevron-right fw' + +%hr.spacer/ + += form_for(@form, url: batch_admin_trends_links_path) do |f| + = hidden_field_tag :page, params[:page] || 1 + + - PreviewCardFilter::KEYS.each do |key| + = hidden_field_tag key, params[key] if params[key].present? + + .batch-table + .batch-table__toolbar + %label.batch-table__toolbar__select.batch-checkbox-all + = check_box_tag :batch_checkbox_all, nil, false + .batch-table__toolbar__actions + = f.button safe_join([fa_icon('check'), t('admin.trends.links.allow')]), name: :approve, class: 'table-action-link', type: :submit, data: { confirm: t('admin.reports.are_you_sure') } + = f.button safe_join([fa_icon('check'), t('admin.trends.links.allow_provider')]), name: :approve_all, class: 'table-action-link', type: :submit, data: { confirm: t('admin.reports.are_you_sure') } + = f.button safe_join([fa_icon('times'), t('admin.trends.links.disallow')]), name: :reject, class: 'table-action-link', type: :submit, data: { confirm: t('admin.reports.are_you_sure') } + = f.button safe_join([fa_icon('times'), t('admin.trends.links.disallow_provider')]), name: :reject_all, class: 'table-action-link', type: :submit, data: { confirm: t('admin.reports.are_you_sure') } + .batch-table__body + - if @preview_cards.empty? + = nothing_here 'nothing-here--under-tabs' + - else + = render partial: 'preview_card', collection: @preview_cards, locals: { f: f } + += paginate @preview_cards diff --git a/app/views/admin/trends/links/preview_card_providers/_preview_card_provider.html.haml b/app/views/admin/trends/links/preview_card_providers/_preview_card_provider.html.haml new file mode 100644 index 000000000..e40e6529d --- /dev/null +++ b/app/views/admin/trends/links/preview_card_providers/_preview_card_provider.html.haml @@ -0,0 +1,16 @@ +.batch-table__row{ class: [preview_card_provider.requires_review? && 'batch-table__row--attention', !preview_card_provider.requires_review? && !preview_card_provider.trendable? && 'batch-table__row--muted'] } + %label.batch-table__row__select.batch-table__row__select--aligned.batch-checkbox + = f.check_box :preview_card_provider_ids, { multiple: true, include_hidden: false }, preview_card_provider.id + + .batch-table__row__content.pending-account + .pending-account__header + %strong= preview_card_provider.domain + + %br/ + + - if preview_card_provider.requires_review? + = t('admin.trends.pending_review') + - elsif preview_card_provider.trendable? + = t('admin.trends.preview_card_providers.allowed') + - else + = t('admin.trends.preview_card_providers.rejected') diff --git a/app/views/admin/trends/links/preview_card_providers/index.html.haml b/app/views/admin/trends/links/preview_card_providers/index.html.haml new file mode 100644 index 000000000..eac6e641f --- /dev/null +++ b/app/views/admin/trends/links/preview_card_providers/index.html.haml @@ -0,0 +1,43 @@ +- content_for :page_title do + = t('admin.trends.preview_card_providers.title') + +- content_for :header_tags do + = javascript_pack_tag 'admin', async: true, crossorigin: 'anonymous' + +.filters + .filter-subset + %strong= t('admin.tags.review') + %ul + %li= filter_link_to t('generic.all'), status: nil + %li= filter_link_to t('admin.trends.approved'), status: 'approved' + %li= filter_link_to t('admin.trends.rejected'), status: 'rejected' + %li= filter_link_to safe_join([t('admin.accounts.moderation.pending'), "(#{PreviewCardProvider.pending_review.count})"], ' '), status: 'pending_review' + .back-link + = link_to admin_trends_links_path do + = fa_icon 'chevron-left fw' + = t('admin.trends.links.title') + + +%hr.spacer/ + += form_for(@form, url: batch_admin_trends_links_preview_card_providers_path) do |f| + = hidden_field_tag :page, params[:page] || 1 + + - PreviewCardProviderFilter::KEYS.each do |key| + = hidden_field_tag key, params[key] if params[key].present? + + .batch-table.optional + .batch-table__toolbar + %label.batch-table__toolbar__select.batch-checkbox-all + = check_box_tag :batch_checkbox_all, nil, false + .batch-table__toolbar__actions + = f.button safe_join([fa_icon('check'), t('admin.trends.allow')]), name: :approve, class: 'table-action-link', type: :submit, data: { confirm: t('admin.reports.are_you_sure') } + = f.button safe_join([fa_icon('times'), t('admin.trends.disallow')]), name: :reject, class: 'table-action-link', type: :submit, data: { confirm: t('admin.reports.are_you_sure') } + + .batch-table__body + - if @preview_card_providers.empty? + = nothing_here 'nothing-here--under-tabs' + - else + = render partial: 'preview_card_provider', collection: @preview_card_providers, locals: { f: f } + += paginate @preview_card_providers diff --git a/app/views/admin/trends/tags/_tag.html.haml b/app/views/admin/trends/tags/_tag.html.haml new file mode 100644 index 000000000..c4af77b00 --- /dev/null +++ b/app/views/admin/trends/tags/_tag.html.haml @@ -0,0 +1,24 @@ +.batch-table__row{ class: [tag.requires_review? && 'batch-table__row--attention', !tag.requires_review? && !tag.trendable? && 'batch-table__row--muted'] } + %label.batch-table__row__select.batch-table__row__select--aligned.batch-checkbox + = f.check_box :tag_ids, { multiple: true, include_hidden: false }, tag.id + + .batch-table__row__content.pending-account + .pending-account__header + = link_to admin_tag_path(tag.id) do + = fa_icon 'hashtag' + = tag.name + + %br/ + + = t('admin.trends.tags.used_by_over_week', count: tag.history.reduce(0) { |sum, day| sum + day.accounts }) + + - if tag.trendable? && (rank = Trends.tags.rank(tag.id)) + • + %abbr{ title: t('admin.trends.tags.current_score', score: Trends.tags.score(tag.id)) }= t('admin.trends.tags.trending_rank', rank: rank + 1) + + - if tag.max_score_at && tag.max_score_at >= Trends::Tags::MAX_SCORE_COOLDOWN.ago && tag.max_score_at < 1.day.ago + • + = t('admin.trends.tags.peaked_on_and_decaying', date: l(tag.max_score_at.to_date, format: :short)) + - elsif tag.requires_review? + • + = t('admin.trends.pending_review') diff --git a/app/views/admin/trends/tags/index.html.haml b/app/views/admin/trends/tags/index.html.haml new file mode 100644 index 000000000..8df0a9920 --- /dev/null +++ b/app/views/admin/trends/tags/index.html.haml @@ -0,0 +1,38 @@ +- content_for :page_title do + = t('admin.trends.tags.title') + +- content_for :header_tags do + = javascript_pack_tag 'admin', async: true, crossorigin: 'anonymous' + +.filters + .filter-subset + %strong= t('admin.tags.review') + %ul + %li= filter_link_to t('generic.all'), status: nil + %li= filter_link_to t('admin.trends.approved'), status: 'approved' + %li= filter_link_to t('admin.trends.rejected'), status: 'rejected' + %li= filter_link_to safe_join([t('admin.accounts.moderation.pending'), "(#{Tag.pending_review.count})"], ' '), status: 'pending_review' + +%hr.spacer/ + += form_for(@form, url: batch_admin_trends_tags_path) do |f| + = hidden_field_tag :page, params[:page] || 1 + + - TagFilter::KEYS.each do |key| + = hidden_field_tag key, params[key] if params[key].present? + + .batch-table.optional + .batch-table__toolbar + %label.batch-table__toolbar__select.batch-checkbox-all + = check_box_tag :batch_checkbox_all, nil, false + .batch-table__toolbar__actions + = f.button safe_join([fa_icon('check'), t('admin.trends.allow')]), name: :approve, class: 'table-action-link', type: :submit, data: { confirm: t('admin.reports.are_you_sure') } + = f.button safe_join([fa_icon('times'), t('admin.trends.disallow')]), name: :reject, class: 'table-action-link', type: :submit, data: { confirm: t('admin.reports.are_you_sure') } + + .batch-table__body + - if @tags.empty? + = nothing_here 'nothing-here--under-tabs' + - else + = render partial: 'tag', collection: @tags, locals: { f: f } + += paginate @tags diff --git a/app/views/admin_mailer/new_trending_links.text.erb b/app/views/admin_mailer/new_trending_links.text.erb new file mode 100644 index 000000000..51789aca5 --- /dev/null +++ b/app/views/admin_mailer/new_trending_links.text.erb @@ -0,0 +1,16 @@ +<%= raw t('application_mailer.salutation', name: display_name(@me)) %> + +<%= raw t('admin_mailer.new_trending_links.body') %> + +<% @links.each do |link| %> +- <%= link.title %> • <%= link.url %> + <%= t('admin.trends.links.usage_comparison', today: link.history.get(Time.now.utc).accounts, yesterday: link.history.get(Time.now.utc - 1.day).accounts) %> • <%= t('admin.trends.tags.current_score', score: Trends.links.score(link.id).round(2)) %> +<% end %> + +<% if @lowest_trending_link %> +<%= t('admin_mailer.new_trending_links.requirements', lowest_link_title: @lowest_trending_link.title, lowest_link_score: Trends.links.score(@lowest_trending_link.id).round(2)) %> +<% else %> +<%= t('admin_mailer.new_trending_links.no_approved_links') %> +<% end %> + +<%= raw t('application_mailer.view')%> <%= admin_trends_links_url %> diff --git a/app/views/admin_mailer/new_trending_tag.text.erb b/app/views/admin_mailer/new_trending_tag.text.erb deleted file mode 100644 index e4bfdc591..000000000 --- a/app/views/admin_mailer/new_trending_tag.text.erb +++ /dev/null @@ -1,5 +0,0 @@ -<%= raw t('application_mailer.salutation', name: display_name(@me)) %> - -<%= raw t('admin_mailer.new_trending_tag.body', name: @tag.name) %> - -<%= raw t('application_mailer.view')%> <%= admin_tags_url(pending_review: '1') %> diff --git a/app/views/admin_mailer/new_trending_tags.text.erb b/app/views/admin_mailer/new_trending_tags.text.erb new file mode 100644 index 000000000..5051e8a96 --- /dev/null +++ b/app/views/admin_mailer/new_trending_tags.text.erb @@ -0,0 +1,16 @@ +<%= raw t('application_mailer.salutation', name: display_name(@me)) %> + +<%= raw t('admin_mailer.new_trending_tags.body') %> + +<% @tags.each do |tag| %> +- #<%= tag.name %> + <%= t('admin.trends.tags.usage_comparison', today: tag.history.get(Time.now.utc).accounts, yesterday: tag.history.get(Time.now.utc - 1.day).accounts) %> • <%= t('admin.trends.tags.current_score', score: Trends.tags.score(tag.id).round(2)) %> +<% end %> + +<% if @lowest_trending_tag %> +<%= t('admin_mailer.new_trending_tags.requirements', lowest_tag_name: @lowest_trending_tag.name, lowest_tag_score: Trends.tags.score(@lowest_trending_tag.id).round(2)) %> +<% else %> +<%= t('admin_mailer.new_trending_tags.no_approved_tags') %> +<% end %> + +<%= raw t('application_mailer.view')%> <%= admin_trends_tags_url(pending_review: '1') %> diff --git a/app/views/application/_sidebar.html.haml b/app/views/application/_sidebar.html.haml index 7ec91c06a..6826c3b58 100644 --- a/app/views/application/_sidebar.html.haml +++ b/app/views/application/_sidebar.html.haml @@ -6,7 +6,7 @@ %p= @instance_presenter.site_short_description.html_safe.presence || t('about.about_mastodon_html') - if Setting.trends && !(user_signed_in? && !current_user.setting_trends) - - trends = TrendingTags.get(3) + - trends = Trends.tags.get(true, 3) - unless trends.empty? .endorsements-widget.trends-widget diff --git a/app/workers/scheduler/trending_tags_scheduler.rb b/app/workers/scheduler/trending_tags_scheduler.rb deleted file mode 100644 index 94d76d010..000000000 --- a/app/workers/scheduler/trending_tags_scheduler.rb +++ /dev/null @@ -1,11 +0,0 @@ -# frozen_string_literal: true - -class Scheduler::TrendingTagsScheduler - include Sidekiq::Worker - - sidekiq_options retry: 0 - - def perform - TrendingTags.update! if Setting.trends - end -end diff --git a/app/workers/scheduler/trends/refresh_scheduler.rb b/app/workers/scheduler/trends/refresh_scheduler.rb new file mode 100644 index 000000000..b559ba46b --- /dev/null +++ b/app/workers/scheduler/trends/refresh_scheduler.rb @@ -0,0 +1,11 @@ +# frozen_string_literal: true + +class Scheduler::Trends::RefreshScheduler + include Sidekiq::Worker + + sidekiq_options retry: 0 + + def perform + Trends.refresh! + end +end diff --git a/app/workers/scheduler/trends/review_notifications_scheduler.rb b/app/workers/scheduler/trends/review_notifications_scheduler.rb new file mode 100644 index 000000000..f334261bd --- /dev/null +++ b/app/workers/scheduler/trends/review_notifications_scheduler.rb @@ -0,0 +1,11 @@ +# frozen_string_literal: true + +class Scheduler::Trends::ReviewNotificationsScheduler + include Sidekiq::Worker + + sidekiq_options retry: 0 + + def perform + Trends.request_review! + end +end -- cgit From 013bee6afb9f76c354e5f8d096d9c85d1ac484ce Mon Sep 17 00:00:00 2001 From: Claire Date: Thu, 25 Nov 2021 23:46:30 +0100 Subject: Fix filtering DMs from non-followed users (#17042) --- app/services/notify_service.rb | 43 +++++++++++++++++++++++++++++++++++- spec/services/notify_service_spec.rb | 17 ++++++++++++-- 2 files changed, 57 insertions(+), 3 deletions(-) (limited to 'app') diff --git a/app/services/notify_service.rb b/app/services/notify_service.rb index a1b5ca1e3..09e28b76b 100644 --- a/app/services/notify_service.rb +++ b/app/services/notify_service.rb @@ -67,8 +67,49 @@ class NotifyService < BaseService message? && @notification.target_status.direct_visibility? end + # Returns true if the sender has been mentionned by the recipient up the thread def response_to_recipient? - @notification.target_status.in_reply_to_account_id == @recipient.id && @notification.target_status.thread&.direct_visibility? + return false if @notification.target_status.in_reply_to_id.nil? + + # Using an SQL CTE to avoid unneeded back-and-forth with SQL server in case of long threads + !Status.count_by_sql([<<-SQL.squish, id: @notification.target_status.in_reply_to_id, recipient_id: @recipient.id, sender_id: @notification.from_account.id]).zero? + WITH RECURSIVE ancestors(id, in_reply_to_id, replying_to_sender) AS ( + SELECT + s.id, s.in_reply_to_id, (CASE + WHEN s.account_id = :recipient_id THEN + EXISTS ( + SELECT * + FROM mentions m + WHERE m.silent = FALSE AND m.account_id = :sender_id AND m.status_id = s.id + ) + ELSE + FALSE + END) + FROM statuses s + WHERE s.id = :id + UNION ALL + SELECT + s.id, + s.in_reply_to_id, + (CASE + WHEN s.account_id = :recipient_id THEN + EXISTS ( + SELECT * + FROM mentions m + WHERE m.silent = FALSE AND m.account_id = :sender_id AND m.status_id = s.id + ) + ELSE + FALSE + END) + FROM ancestors st + JOIN statuses s ON s.id = st.in_reply_to_id + WHERE st.replying_to_sender IS FALSE + ) + SELECT COUNT(*) + FROM ancestors st + JOIN statuses s ON s.id = st.id + WHERE st.replying_to_sender IS TRUE AND s.visibility = 3 + SQL end def from_staff? diff --git a/spec/services/notify_service_spec.rb b/spec/services/notify_service_spec.rb index f2cb22c5e..7433866b7 100644 --- a/spec/services/notify_service_spec.rb +++ b/spec/services/notify_service_spec.rb @@ -64,8 +64,9 @@ RSpec.describe NotifyService, type: :service do is_expected.to_not change(Notification, :count) end - context 'if the message chain initiated by recipient, but is not direct message' do + context 'if the message chain is initiated by recipient, but is not direct message' do let(:reply_to) { Fabricate(:status, account: recipient) } + let!(:mention) { Fabricate(:mention, account: sender, status: reply_to) } let(:activity) { Fabricate(:mention, account: recipient, status: Fabricate(:status, account: sender, visibility: :direct, thread: reply_to)) } it 'does not notify' do @@ -73,8 +74,20 @@ RSpec.describe NotifyService, type: :service do end end - context 'if the message chain initiated by recipient and is direct message' do + context 'if the message chain is initiated by recipient, but without a mention to the sender, even if the sender sends multiple messages in a row' do + let(:reply_to) { Fabricate(:status, account: recipient) } + let!(:mention) { Fabricate(:mention, account: sender, status: reply_to) } + let(:dummy_reply) { Fabricate(:status, account: sender, visibility: :direct, thread: reply_to) } + let(:activity) { Fabricate(:mention, account: recipient, status: Fabricate(:status, account: sender, visibility: :direct, thread: dummy_reply)) } + + it 'does not notify' do + is_expected.to_not change(Notification, :count) + end + end + + context 'if the message chain is initiated by the recipient with a mention to the sender' do let(:reply_to) { Fabricate(:status, account: recipient, visibility: :direct) } + let!(:mention) { Fabricate(:mention, account: sender, status: reply_to) } let(:activity) { Fabricate(:mention, account: recipient, status: Fabricate(:status, account: sender, visibility: :direct, thread: reply_to)) } it 'does notify' do -- cgit From b1fd6d44901a13450d22884b02eb6e9ae4fc1248 Mon Sep 17 00:00:00 2001 From: Claire Date: Thu, 25 Nov 2021 23:46:39 +0100 Subject: Fix handling of recursive toots in WebUI (#17041) --- app/javascript/mastodon/features/status/index.js | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) (limited to 'app') diff --git a/app/javascript/mastodon/features/status/index.js b/app/javascript/mastodon/features/status/index.js index 055963d14..f342a3641 100644 --- a/app/javascript/mastodon/features/status/index.js +++ b/app/javascript/mastodon/features/status/index.js @@ -83,7 +83,7 @@ const makeMapStateToProps = () => { ancestorsIds = ancestorsIds.withMutations(mutable => { let id = statusId; - while (id) { + while (id && !mutable.includes(id)) { mutable.unshift(id); id = inReplyTos.get(id); } @@ -101,7 +101,7 @@ const makeMapStateToProps = () => { const ids = [statusId]; while (ids.length > 0) { - let id = ids.shift(); + let id = ids.pop(); const replies = contextReplies.get(id); if (statusId !== id) { @@ -110,7 +110,7 @@ const makeMapStateToProps = () => { if (replies) { replies.reverse().forEach(reply => { - ids.unshift(reply); + if (!ids.includes(reply) && !descendantsIds.includes(reply) && statusId !== reply) ids.push(reply); }); } } -- cgit From 67c4e8e317aabef88e7633c9a5534613d9363983 Mon Sep 17 00:00:00 2001 From: Claire Date: Thu, 25 Nov 2021 23:46:39 +0100 Subject: [Glitch] Fix handling of recursive toots in WebUI Port b1fd6d44901a13450d22884b02eb6e9ae4fc1248 to glitch-soc Signed-off-by: Claire --- app/javascript/flavours/glitch/features/status/index.js | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) (limited to 'app') diff --git a/app/javascript/flavours/glitch/features/status/index.js b/app/javascript/flavours/glitch/features/status/index.js index bddf1a2ef..12ea407ad 100644 --- a/app/javascript/flavours/glitch/features/status/index.js +++ b/app/javascript/flavours/glitch/features/status/index.js @@ -70,7 +70,7 @@ const makeMapStateToProps = () => { ancestorsIds = ancestorsIds.withMutations(mutable => { let id = statusId; - while (id) { + while (id && !mutable.includes(id)) { mutable.unshift(id); id = inReplyTos.get(id); } @@ -88,7 +88,7 @@ const makeMapStateToProps = () => { const ids = [statusId]; while (ids.length > 0) { - let id = ids.shift(); + let id = ids.pop(); const replies = contextReplies.get(id); if (statusId !== id) { @@ -97,7 +97,7 @@ const makeMapStateToProps = () => { if (replies) { replies.reverse().forEach(reply => { - ids.unshift(reply); + if (!ids.includes(reply) && !descendantsIds.includes(reply) && statusId !== reply) ids.push(reply); }); } } -- cgit From 0ccaedc92bff79fa2922c700dd7dc42378b99090 Mon Sep 17 00:00:00 2001 From: Claire Date: Thu, 25 Nov 2021 23:53:28 +0100 Subject: Disable trending links review request emails --- app/models/trends.rb | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'app') diff --git a/app/models/trends.rb b/app/models/trends.rb index 7dd3a9c87..8f8cb0261 100644 --- a/app/models/trends.rb +++ b/app/models/trends.rb @@ -18,7 +18,7 @@ module Trends end def self.request_review! - [links, tags].each(&:request_review) if enabled? + [tags].each(&:request_review) if enabled? end def self.enabled? -- cgit From 5f10e64330635bfd609ba5acdd78fa505c12f5b1 Mon Sep 17 00:00:00 2001 From: Claire Date: Fri, 26 Nov 2021 00:26:06 +0100 Subject: Fix trends admin page crashing --- app/models/trends/base.rb | 13 ------------- app/models/trends/links.rb | 25 +++++++++++++++---------- app/models/trends/tags.rb | 25 +++++++++++++++---------- spec/models/trends/tags_spec.rb | 9 +++++++-- 4 files changed, 37 insertions(+), 35 deletions(-) (limited to 'app') diff --git a/app/models/trends/base.rb b/app/models/trends/base.rb index b767dcb1a..788f128a0 100644 --- a/app/models/trends/base.rb +++ b/app/models/trends/base.rb @@ -3,19 +3,6 @@ class Trends::Base include Redisable - class_attribute :default_options - - attr_reader :options - - # @param [Hash] options - # @option options [Integer] :threshold Minimum amount of uses by unique accounts to begin calculating the score - # @option options [Integer] :review_threshold Minimum rank (lower = better) before requesting a review - # @option options [ActiveSupport::Duration] :max_score_cooldown For this amount of time, the peak score (if bigger than current score) is decayed-from - # @option options [ActiveSupport::Duration] :max_score_halflife How quickly a peak score decays - def initialize(options = {}) - @options = self.class.default_options.merge(options) - end - def register(_status) raise NotImplementedError end diff --git a/app/models/trends/links.rb b/app/models/trends/links.rb index a0d65138b..a7cd9e29d 100644 --- a/app/models/trends/links.rb +++ b/app/models/trends/links.rb @@ -3,12 +3,17 @@ class Trends::Links < Trends::Base PREFIX = 'trending_links' - self.default_options = { - threshold: 15, - review_threshold: 10, - max_score_cooldown: 2.days.freeze, - max_score_halflife: 8.hours.freeze, - } + # Minimum amount of uses by unique accounts to begin calculating the score + THRESHOLD = 15 + + # Minimum rank (lower = better) before requesting a review + REVIEW_THRESHOLD = 10 + + # For this amount of time, the peak score (if bigger than current score) is decayed-from + MAX_SCORE_COOLDOWN = 2.days.freeze + + # How quickly a peak score decays + MAX_SCORE_HALFLIFE = 8.hours.freeze def register(status, at_time = Time.now.utc) original_status = status.reblog? ? status.reblog : status @@ -76,10 +81,10 @@ class Trends::Links < Trends::Base observed = preview_card.history.get(at_time).accounts.to_f max_time = preview_card.max_score_at max_score = preview_card.max_score - max_score = 0 if max_time.nil? || max_time < (at_time - options[:max_score_cooldown]) + max_score = 0 if max_time.nil? || max_time < (at_time - MAX_SCORE_COOLDOWN) score = begin - if expected > observed || observed < options[:threshold] + if expected > observed || observed < THRESHOLD 0 else ((observed - expected)**2) / expected @@ -94,7 +99,7 @@ class Trends::Links < Trends::Base preview_card.update_columns(max_score: max_score, max_score_at: max_time) end - decaying_score = max_score * (0.5**((at_time.to_f - max_time.to_f) / options[:max_score_halflife].to_f)) + decaying_score = max_score * (0.5**((at_time.to_f - max_time.to_f) / MAX_SCORE_HALFLIFE.to_f)) if decaying_score.zero? redis.zrem("#{PREFIX}:all", preview_card.id) @@ -112,6 +117,6 @@ class Trends::Links < Trends::Base end def would_be_trending?(id) - score(id) > score_at_rank(options[:review_threshold] - 1) + score(id) > score_at_rank(REVIEW_THRESHOLD - 1) end end diff --git a/app/models/trends/tags.rb b/app/models/trends/tags.rb index 13e0ab56b..027e3dbac 100644 --- a/app/models/trends/tags.rb +++ b/app/models/trends/tags.rb @@ -3,12 +3,17 @@ class Trends::Tags < Trends::Base PREFIX = 'trending_tags' - self.default_options = { - threshold: 15, - review_threshold: 10, - max_score_cooldown: 2.days.freeze, - max_score_halflife: 4.hours.freeze, - } + # Minimum amount of uses by unique accounts to begin calculating the score + THRESHOLD = 15 + + # Minimum rank (lower = better) before requesting a review + REVIEW_THRESHOLD = 10 + + # For this amount of time, the peak score (if bigger than current score) is decayed-from + MAX_SCORE_COOLDOWN = 2.days.freeze + + # How quickly a peak score decays + MAX_SCORE_HALFLIFE = 4.hours.freeze def register(status, at_time = Time.now.utc) original_status = status.reblog? ? status.reblog : status @@ -70,10 +75,10 @@ class Trends::Tags < Trends::Base observed = tag.history.get(at_time).accounts.to_f max_time = tag.max_score_at max_score = tag.max_score - max_score = 0 if max_time.nil? || max_time < (at_time - options[:max_score_cooldown]) + max_score = 0 if max_time.nil? || max_time < (at_time - MAX_SCORE_COOLDOWN) score = begin - if expected > observed || observed < options[:threshold] + if expected > observed || observed < THRESHOLD 0 else ((observed - expected)**2) / expected @@ -88,7 +93,7 @@ class Trends::Tags < Trends::Base tag.update_columns(max_score: max_score, max_score_at: max_time) end - decaying_score = max_score * (0.5**((at_time.to_f - max_time.to_f) / options[:max_score_halflife].to_f)) + decaying_score = max_score * (0.5**((at_time.to_f - max_time.to_f) / MAX_SCORE_HALFLIFE.to_f)) if decaying_score.zero? redis.zrem("#{PREFIX}:all", tag.id) @@ -106,6 +111,6 @@ class Trends::Tags < Trends::Base end def would_be_trending?(id) - score(id) > score_at_rank(options[:review_threshold] - 1) + score(id) > score_at_rank(REVIEW_THRESHOLD - 1) end end diff --git a/spec/models/trends/tags_spec.rb b/spec/models/trends/tags_spec.rb index 4f98c6aa4..23f8a7ba7 100644 --- a/spec/models/trends/tags_spec.rb +++ b/spec/models/trends/tags_spec.rb @@ -1,10 +1,15 @@ require 'rails_helper' RSpec.describe Trends::Tags do - subject { described_class.new(threshold: 5, review_threshold: 10) } + subject { described_class.new } let!(:at_time) { DateTime.new(2021, 11, 14, 10, 15, 0) } + before do + stub_const 'Trends::Tags::THRESHOLD', 5 + stub_const 'Trends::Tags::REVIEW_THRESHOLD', 10 + end + describe '#add' do let(:tag) { Fabricate(:tag) } @@ -59,7 +64,7 @@ RSpec.describe Trends::Tags do subject.refresh(yesterday + 12.hours) original_score = subject.score(tag3.id) expect(original_score).to eq 144.0 - subject.refresh(yesterday + 12.hours + subject.options[:max_score_halflife]) + subject.refresh(yesterday + 12.hours + described_class::MAX_SCORE_HALFLIFE) decayed_score = subject.score(tag3.id) expect(decayed_score).to be <= original_score / 2 end -- cgit