From 619fad6cf8078ea997554081febe850404bee73c Mon Sep 17 00:00:00 2001 From: Eugen Rochko Date: Sun, 11 Apr 2021 11:22:50 +0200 Subject: Remove spam check and dependency on nilsimsa gem (#16011) --- app/controllers/admin/dashboard_controller.rb | 1 - 1 file changed, 1 deletion(-) (limited to 'app/controllers') diff --git a/app/controllers/admin/dashboard_controller.rb b/app/controllers/admin/dashboard_controller.rb index 4422825ee..c829ed98f 100644 --- a/app/controllers/admin/dashboard_controller.rb +++ b/app/controllers/admin/dashboard_controller.rb @@ -35,7 +35,6 @@ module Admin @whitelist_enabled = whitelist_mode? @profile_directory = Setting.profile_directory @timeline_preview = Setting.timeline_preview - @spam_check_enabled = Setting.spam_check_enabled @trends_enabled = Setting.trends end -- cgit From f7117646afddb2676e9275d8efe90c3a20c59021 Mon Sep 17 00:00:00 2001 From: Eugen Rochko Date: Mon, 12 Apr 2021 12:37:14 +0200 Subject: Add cold-start follow recommendations (#15945) --- .../admin/follow_recommendations_controller.rb | 53 ++++++++++++++++++ app/controllers/api/v1/suggestions_controller.rb | 2 +- app/controllers/api/v2/suggestions_controller.rb | 19 +++++++ app/javascript/mastodon/actions/suggestions.js | 8 +-- .../features/compose/components/search_results.js | 12 ++--- app/javascript/mastodon/reducers/suggestions.js | 8 +-- app/lib/potential_friendship_tracker.rb | 12 +++-- app/models/account.rb | 3 +- app/models/account_suggestions.rb | 17 ++++++ app/models/account_summary.rb | 25 +++++++++ app/models/concerns/account_associations.rb | 3 ++ app/models/follow_recommendation.rb | 39 ++++++++++++++ app/models/follow_recommendation_filter.rb | 26 +++++++++ app/models/follow_recommendation_suppression.rb | 28 ++++++++++ app/models/form/account_batch.rb | 18 +++++++ app/policies/follow_recommendation_policy.rb | 15 ++++++ app/serializers/rest/suggestion_serializer.rb | 7 +++ .../follow_recommendations/_account.html.haml | 20 +++++++ .../admin/follow_recommendations/show.html.haml | 42 +++++++++++++++ .../scheduler/follow_recommendations_scheduler.rb | 61 +++++++++++++++++++++ config/locales/en.yml | 8 +++ config/navigation.rb | 1 + config/routes.rb | 2 + config/sidekiq.yml | 4 ++ .../20210322164601_create_account_summaries.rb | 9 ++++ ...20210323114347_create_follow_recommendations.rb | 5 ++ ...13_create_follow_recommendation_suppressions.rb | 9 ++++ db/schema.rb | 63 +++++++++++++++++++--- db/views/account_summaries_v01.sql | 22 ++++++++ db/views/follow_recommendations_v01.sql | 38 +++++++++++++ ...follow_recommendation_suppression_fabricator.rb | 3 ++ .../follow_recommendation_suppression_spec.rb | 4 ++ 32 files changed, 560 insertions(+), 26 deletions(-) create mode 100644 app/controllers/admin/follow_recommendations_controller.rb create mode 100644 app/controllers/api/v2/suggestions_controller.rb create mode 100644 app/models/account_suggestions.rb create mode 100644 app/models/account_summary.rb create mode 100644 app/models/follow_recommendation.rb create mode 100644 app/models/follow_recommendation_filter.rb create mode 100644 app/models/follow_recommendation_suppression.rb create mode 100644 app/policies/follow_recommendation_policy.rb create mode 100644 app/serializers/rest/suggestion_serializer.rb create mode 100644 app/views/admin/follow_recommendations/_account.html.haml create mode 100644 app/views/admin/follow_recommendations/show.html.haml create mode 100644 app/workers/scheduler/follow_recommendations_scheduler.rb create mode 100644 db/migrate/20210322164601_create_account_summaries.rb create mode 100644 db/migrate/20210323114347_create_follow_recommendations.rb create mode 100644 db/migrate/20210324171613_create_follow_recommendation_suppressions.rb create mode 100644 db/views/account_summaries_v01.sql create mode 100644 db/views/follow_recommendations_v01.sql create mode 100644 spec/fabricators/follow_recommendation_suppression_fabricator.rb create mode 100644 spec/models/follow_recommendation_suppression_spec.rb (limited to 'app/controllers') diff --git a/app/controllers/admin/follow_recommendations_controller.rb b/app/controllers/admin/follow_recommendations_controller.rb new file mode 100644 index 000000000..e3eac62b3 --- /dev/null +++ b/app/controllers/admin/follow_recommendations_controller.rb @@ -0,0 +1,53 @@ +# frozen_string_literal: true + +module Admin + class FollowRecommendationsController < BaseController + before_action :set_language + + def show + authorize :follow_recommendation, :show? + + @form = Form::AccountBatch.new + @accounts = filtered_follow_recommendations + end + + def update + @form = Form::AccountBatch.new(form_account_batch_params.merge(current_account: current_account, action: action_from_button)) + @form.save + rescue ActionController::ParameterMissing + # Do nothing + ensure + redirect_to admin_follow_recommendations_path(filter_params) + end + + private + + def set_language + @language = follow_recommendation_filter.language + end + + def filtered_follow_recommendations + follow_recommendation_filter.results + end + + def follow_recommendation_filter + @follow_recommendation_filter ||= FollowRecommendationFilter.new(filter_params) + end + + def form_account_batch_params + params.require(:form_account_batch).permit(:action, account_ids: []) + end + + def filter_params + params.slice(*FollowRecommendationFilter::KEYS).permit(*FollowRecommendationFilter::KEYS) + end + + def action_from_button + if params[:suppress] + 'suppress_follow_recommendation' + elsif params[:unsuppress] + 'unsuppress_follow_recommendation' + end + end + end +end diff --git a/app/controllers/api/v1/suggestions_controller.rb b/app/controllers/api/v1/suggestions_controller.rb index 52054160d..b2788cc76 100644 --- a/app/controllers/api/v1/suggestions_controller.rb +++ b/app/controllers/api/v1/suggestions_controller.rb @@ -19,6 +19,6 @@ class Api::V1::SuggestionsController < Api::BaseController private def set_accounts - @accounts = PotentialFriendshipTracker.get(current_account.id, limit: limit_param(DEFAULT_ACCOUNTS_LIMIT)) + @accounts = PotentialFriendshipTracker.get(current_account, limit_param(DEFAULT_ACCOUNTS_LIMIT)) end end diff --git a/app/controllers/api/v2/suggestions_controller.rb b/app/controllers/api/v2/suggestions_controller.rb new file mode 100644 index 000000000..35eb276c0 --- /dev/null +++ b/app/controllers/api/v2/suggestions_controller.rb @@ -0,0 +1,19 @@ +# frozen_string_literal: true + +class Api::V2::SuggestionsController < Api::BaseController + include Authorization + + before_action -> { doorkeeper_authorize! :read } + before_action :require_user! + before_action :set_suggestions + + def index + render json: @suggestions, each_serializer: REST::SuggestionSerializer + end + + private + + def set_suggestions + @suggestions = AccountSuggestions.get(current_account, limit_param(DEFAULT_ACCOUNTS_LIMIT)) + end +end diff --git a/app/javascript/mastodon/actions/suggestions.js b/app/javascript/mastodon/actions/suggestions.js index b15bd916b..0bf959017 100644 --- a/app/javascript/mastodon/actions/suggestions.js +++ b/app/javascript/mastodon/actions/suggestions.js @@ -11,8 +11,8 @@ export function fetchSuggestions() { return (dispatch, getState) => { dispatch(fetchSuggestionsRequest()); - api(getState).get('/api/v1/suggestions').then(response => { - dispatch(importFetchedAccounts(response.data)); + api(getState).get('/api/v2/suggestions').then(response => { + dispatch(importFetchedAccounts(response.data.map(x => x.account))); dispatch(fetchSuggestionsSuccess(response.data)); }).catch(error => dispatch(fetchSuggestionsFail(error))); }; @@ -25,10 +25,10 @@ export function fetchSuggestionsRequest() { }; }; -export function fetchSuggestionsSuccess(accounts) { +export function fetchSuggestionsSuccess(suggestions) { return { type: SUGGESTIONS_FETCH_SUCCESS, - accounts, + suggestions, skipLoading: true, }; }; diff --git a/app/javascript/mastodon/features/compose/components/search_results.js b/app/javascript/mastodon/features/compose/components/search_results.js index 4b4cdff74..c4e160b8a 100644 --- a/app/javascript/mastodon/features/compose/components/search_results.js +++ b/app/javascript/mastodon/features/compose/components/search_results.js @@ -51,13 +51,13 @@ class SearchResults extends ImmutablePureComponent { - {suggestions && suggestions.map(accountId => ( + {suggestions && suggestions.map(suggestion => ( ))} diff --git a/app/javascript/mastodon/reducers/suggestions.js b/app/javascript/mastodon/reducers/suggestions.js index 834be728f..1a6e66ee7 100644 --- a/app/javascript/mastodon/reducers/suggestions.js +++ b/app/javascript/mastodon/reducers/suggestions.js @@ -19,18 +19,18 @@ export default function suggestionsReducer(state = initialState, action) { return state.set('isLoading', true); case SUGGESTIONS_FETCH_SUCCESS: return state.withMutations(map => { - map.set('items', fromJS(action.accounts.map(x => x.id))); + map.set('items', fromJS(action.suggestions.map(x => ({ ...x, account: x.account.id })))); map.set('isLoading', false); }); case SUGGESTIONS_FETCH_FAIL: return state.set('isLoading', false); case SUGGESTIONS_DISMISS: - return state.update('items', list => list.filterNot(id => id === action.id)); + return state.update('items', list => list.filterNot(x => x.account === action.id)); case ACCOUNT_BLOCK_SUCCESS: case ACCOUNT_MUTE_SUCCESS: - return state.update('items', list => list.filterNot(id => id === action.relationship.id)); + return state.update('items', list => list.filterNot(x => x.account === action.relationship.id)); case DOMAIN_BLOCK_SUCCESS: - return state.update('items', list => list.filterNot(id => action.accounts.includes(id))); + return state.update('items', list => list.filterNot(x => action.accounts.includes(x.account))); default: return state; } diff --git a/app/lib/potential_friendship_tracker.rb b/app/lib/potential_friendship_tracker.rb index 188aa4a27..e72d454b6 100644 --- a/app/lib/potential_friendship_tracker.rb +++ b/app/lib/potential_friendship_tracker.rb @@ -28,10 +28,14 @@ class PotentialFriendshipTracker redis.zrem("interactions:#{account_id}", target_account_id) end - def get(account_id, limit: 20, offset: 0) - account_ids = redis.zrevrange("interactions:#{account_id}", offset, limit) - return [] if account_ids.empty? - Account.searchable.where(id: account_ids) + def get(account, limit) + account_ids = redis.zrevrange("interactions:#{account.id}", 0, limit) + + return [] if account_ids.empty? || limit < 1 + + accounts = Account.searchable.where(id: account_ids).index_by(&:id) + + account_ids.map { |id| accounts[id.to_i] }.compact end end end diff --git a/app/models/account.rb b/app/models/account.rb index d85fd1f6e..80689d4aa 100644 --- a/app/models/account.rb +++ b/app/models/account.rb @@ -110,6 +110,7 @@ class Account < ApplicationRecord scope :matches_domain, ->(value) { where(arel_table[:domain].matches("%#{value}%")) } scope :searchable, -> { without_suspended.where(moved_to_account_id: nil) } scope :discoverable, -> { searchable.without_silenced.where(discoverable: true).left_outer_joins(:account_stat) } + scope :followable_by, ->(account) { joins(arel_table.join(Follow.arel_table, Arel::Nodes::OuterJoin).on(arel_table[:id].eq(Follow.arel_table[:target_account_id]).and(Follow.arel_table[:account_id].eq(account.id))).join_sources).where(Follow.arel_table[:id].eq(nil)).joins(arel_table.join(FollowRequest.arel_table, Arel::Nodes::OuterJoin).on(arel_table[:id].eq(FollowRequest.arel_table[:target_account_id]).and(FollowRequest.arel_table[:account_id].eq(account.id))).join_sources).where(FollowRequest.arel_table[:id].eq(nil)) } scope :tagged_with, ->(tag) { joins(:accounts_tags).where(accounts_tags: { tag_id: tag }) } scope :by_recent_status, -> { order(Arel.sql('(case when account_stats.last_status_at is null then 1 else 0 end) asc, account_stats.last_status_at desc, accounts.id desc')) } scope :by_recent_sign_in, -> { order(Arel.sql('(case when users.current_sign_in_at is null then 1 else 0 end) asc, users.current_sign_in_at desc, accounts.id desc')) } @@ -363,7 +364,7 @@ class Account < ApplicationRecord end def excluded_from_timeline_account_ids - Rails.cache.fetch("exclude_account_ids_for:#{id}") { blocking.pluck(:target_account_id) + blocked_by.pluck(:account_id) + muting.pluck(:target_account_id) } + Rails.cache.fetch("exclude_account_ids_for:#{id}") { block_relationships.pluck(:target_account_id) + blocked_by_relationships.pluck(:account_id) + mute_relationships.pluck(:target_account_id) } end def excluded_from_timeline_domains diff --git a/app/models/account_suggestions.rb b/app/models/account_suggestions.rb new file mode 100644 index 000000000..7fe9d618e --- /dev/null +++ b/app/models/account_suggestions.rb @@ -0,0 +1,17 @@ +# frozen_string_literal: true + +class AccountSuggestions + class Suggestion < ActiveModelSerializers::Model + attributes :account, :source + end + + def self.get(account, limit) + suggestions = PotentialFriendshipTracker.get(account, limit).map { |target_account| Suggestion.new(account: target_account, source: :past_interaction) } + suggestions.concat(FollowRecommendation.get(account, limit - suggestions.size, suggestions.map { |suggestion| suggestion.account.id }).map { |target_account| Suggestion.new(account: target_account, source: :global) }) if suggestions.size < limit + suggestions + end + + def self.remove(account, target_account_id) + PotentialFriendshipTracker.remove(account.id, target_account_id) + end +end diff --git a/app/models/account_summary.rb b/app/models/account_summary.rb new file mode 100644 index 000000000..6a7e17c6c --- /dev/null +++ b/app/models/account_summary.rb @@ -0,0 +1,25 @@ +# frozen_string_literal: true +# == Schema Information +# +# Table name: account_summaries +# +# account_id :bigint(8) primary key +# language :string +# sensitive :boolean +# + +class AccountSummary < ApplicationRecord + self.primary_key = :account_id + + scope :safe, -> { where(sensitive: false) } + scope :localized, ->(locale) { where(language: locale) } + scope :filtered, -> { joins(arel_table.join(FollowRecommendationSuppression.arel_table, Arel::Nodes::OuterJoin).on(arel_table[:account_id].eq(FollowRecommendationSuppression.arel_table[:account_id])).join_sources).where(FollowRecommendationSuppression.arel_table[:id].eq(nil)) } + + def self.refresh + Scenic.database.refresh_materialized_view(table_name, concurrently: true, cascade: false) + end + + def readonly? + true + end +end diff --git a/app/models/concerns/account_associations.rb b/app/models/concerns/account_associations.rb index 98849f8fc..aaf371ebd 100644 --- a/app/models/concerns/account_associations.rb +++ b/app/models/concerns/account_associations.rb @@ -63,5 +63,8 @@ module AccountAssociations # Account deletion requests has_one :deletion_request, class_name: 'AccountDeletionRequest', inverse_of: :account, dependent: :destroy + + # Follow recommendations + has_one :follow_recommendation_suppression, inverse_of: :account, dependent: :destroy end end diff --git a/app/models/follow_recommendation.rb b/app/models/follow_recommendation.rb new file mode 100644 index 000000000..c4355224d --- /dev/null +++ b/app/models/follow_recommendation.rb @@ -0,0 +1,39 @@ +# frozen_string_literal: true +# == Schema Information +# +# Table name: follow_recommendations +# +# account_id :bigint(8) primary key +# rank :decimal(, ) +# reason :text is an Array +# + +class FollowRecommendation < ApplicationRecord + self.primary_key = :account_id + + belongs_to :account_summary, foreign_key: :account_id + belongs_to :account, foreign_key: :account_id + + scope :safe, -> { joins(:account_summary).merge(AccountSummary.safe) } + scope :localized, ->(locale) { joins(:account_summary).merge(AccountSummary.localized(locale)) } + scope :filtered, -> { joins(:account_summary).merge(AccountSummary.filtered) } + + def readonly? + true + end + + def self.get(account, limit, exclude_account_ids = []) + account_ids = Redis.current.zrevrange("follow_recommendations:#{account.user_locale}", 0, -1).map(&:to_i) - exclude_account_ids - [account.id] + + return [] if account_ids.empty? || limit < 1 + + accounts = Account.followable_by(account) + .not_excluded_by_account(account) + .not_domain_blocked_by_account(account) + .where(id: account_ids) + .limit(limit) + .index_by(&:id) + + account_ids.map { |id| accounts[id] }.compact + end +end diff --git a/app/models/follow_recommendation_filter.rb b/app/models/follow_recommendation_filter.rb new file mode 100644 index 000000000..acf03cd84 --- /dev/null +++ b/app/models/follow_recommendation_filter.rb @@ -0,0 +1,26 @@ +# frozen_string_literal: true + +class FollowRecommendationFilter + KEYS = %i( + language + status + ).freeze + + attr_reader :params, :language + + def initialize(params) + @language = params.delete('language') || I18n.locale + @params = params + end + + def results + if params['status'] == 'suppressed' + Account.joins(:follow_recommendation_suppression).order(FollowRecommendationSuppression.arel_table[:id].desc).to_a + else + account_ids = Redis.current.zrevrange("follow_recommendations:#{@language}", 0, -1).map(&:to_i) + accounts = Account.where(id: account_ids).index_by(&:id) + + account_ids.map { |id| accounts[id] }.compact + end + end +end diff --git a/app/models/follow_recommendation_suppression.rb b/app/models/follow_recommendation_suppression.rb new file mode 100644 index 000000000..170506b85 --- /dev/null +++ b/app/models/follow_recommendation_suppression.rb @@ -0,0 +1,28 @@ +# frozen_string_literal: true +# == Schema Information +# +# Table name: follow_recommendation_suppressions +# +# id :bigint(8) not null, primary key +# account_id :bigint(8) not null +# created_at :datetime not null +# updated_at :datetime not null +# + +class FollowRecommendationSuppression < ApplicationRecord + include Redisable + + belongs_to :account + + after_commit :remove_follow_recommendations, on: :create + + private + + def remove_follow_recommendations + redis.pipelined do + I18n.available_locales.each do |locale| + redis.zrem("follow_recommendations:#{locale}", account_id) + end + end + end +end diff --git a/app/models/form/account_batch.rb b/app/models/form/account_batch.rb index 26d6d3abf..698933c9f 100644 --- a/app/models/form/account_batch.rb +++ b/app/models/form/account_batch.rb @@ -21,6 +21,10 @@ class Form::AccountBatch approve! when 'reject' reject! + when 'suppress_follow_recommendation' + suppress_follow_recommendation! + when 'unsuppress_follow_recommendation' + unsuppress_follow_recommendation! end end @@ -79,4 +83,18 @@ class Form::AccountBatch records.each { |account| authorize(account.user, :reject?) } .each { |account| DeleteAccountService.new.call(account, reserve_email: false, reserve_username: false) } end + + def suppress_follow_recommendation! + authorize(:follow_recommendation, :suppress?) + + accounts.each do |account| + FollowRecommendationSuppression.create(account: account) + end + end + + def unsuppress_follow_recommendation! + authorize(:follow_recommendation, :unsuppress?) + + FollowRecommendationSuppression.where(account_id: account_ids).destroy_all + end end diff --git a/app/policies/follow_recommendation_policy.rb b/app/policies/follow_recommendation_policy.rb new file mode 100644 index 000000000..68cd0e547 --- /dev/null +++ b/app/policies/follow_recommendation_policy.rb @@ -0,0 +1,15 @@ +# frozen_string_literal: true + +class FollowRecommendationPolicy < ApplicationPolicy + def show? + staff? + end + + def suppress? + staff? + end + + def unsuppress? + staff? + end +end diff --git a/app/serializers/rest/suggestion_serializer.rb b/app/serializers/rest/suggestion_serializer.rb new file mode 100644 index 000000000..3d697fd9f --- /dev/null +++ b/app/serializers/rest/suggestion_serializer.rb @@ -0,0 +1,7 @@ +# frozen_string_literal: true + +class REST::SuggestionSerializer < ActiveModel::Serializer + attributes :source + + has_one :account, serializer: REST::AccountSerializer +end diff --git a/app/views/admin/follow_recommendations/_account.html.haml b/app/views/admin/follow_recommendations/_account.html.haml new file mode 100644 index 000000000..af5a4aaf7 --- /dev/null +++ b/app/views/admin/follow_recommendations/_account.html.haml @@ -0,0 +1,20 @@ +.batch-table__row + %label.batch-table__row__select.batch-table__row__select--aligned.batch-checkbox + = f.check_box :account_ids, { multiple: true, include_hidden: false }, account.id + .batch-table__row__content.batch-table__row__content--unpadded + %table.accounts-table + %tbody + %tr + %td= account_link_to account + %td.accounts-table__count.optional + = number_to_human account.statuses_count, strip_insignificant_zeros: true + %small= t('accounts.posts', count: account.statuses_count).downcase + %td.accounts-table__count.optional + = number_to_human account.followers_count, strip_insignificant_zeros: true + %small= t('accounts.followers', count: account.followers_count).downcase + %td.accounts-table__count + - if account.last_status_at.present? + %time.time-ago{ datetime: account.last_status_at.to_date.iso8601, title: l(account.last_status_at.to_date) }= l account.last_status_at + - else + \- + %small= t('accounts.last_active') diff --git a/app/views/admin/follow_recommendations/show.html.haml b/app/views/admin/follow_recommendations/show.html.haml new file mode 100644 index 000000000..1f050329a --- /dev/null +++ b/app/views/admin/follow_recommendations/show.html.haml @@ -0,0 +1,42 @@ +- content_for :page_title do + = t('admin.follow_recommendations.title') + +- content_for :header_tags do + = javascript_pack_tag 'admin', async: true, crossorigin: 'anonymous' + +.simple_form + %p.hint= t('admin.follow_recommendations.description_html') + +%hr.spacer/ + += form_tag admin_follow_recommendations_path, method: 'GET', class: 'simple_form' do + .filters + .filter-subset.filter-subset--with-select + %strong= t('admin.follow_recommendations.language') + .input.select.optional + = select_tag :language, options_for_select(I18n.available_locales.map { |key| [human_locale(key), key]}, @language) + + .filter-subset + %strong= t('admin.follow_recommendations.status') + %ul + %li= filter_link_to t('admin.accounts.moderation.active'), status: nil + %li= filter_link_to t('admin.follow_recommendations.suppressed'), status: 'suppressed' + += form_for(@form, url: admin_follow_recommendations_path, method: :patch) do |f| + - RelationshipFilter::KEYS.each do |key| + = hidden_field_tag key, params[key] if params[key].present? + + .batch-table + .batch-table__toolbar + %label.batch-table__toolbar__select.batch-checkbox-all + = check_box_tag :batch_checkbox_all, nil, false + .batch-table__toolbar__actions + - if params[:status].blank? && can?(:suppress, :follow_recommendation) + = f.button safe_join([fa_icon('times'), t('admin.follow_recommendations.suppress')]), name: :suppress, class: 'table-action-link', type: :submit, data: { confirm: t('admin.reports.are_you_sure') } + - if params[:status] == 'suppressed' && can?(:unsuppress, :follow_recommendation) + = f.button safe_join([fa_icon('plus'), t('admin.follow_recommendations.unsuppress')]), name: :unsuppress, class: 'table-action-link', type: :submit + .batch-table__body + - if @accounts.empty? + = nothing_here 'nothing-here--under-tabs' + - else + = render partial: 'account', collection: @accounts, locals: { f: f } diff --git a/app/workers/scheduler/follow_recommendations_scheduler.rb b/app/workers/scheduler/follow_recommendations_scheduler.rb new file mode 100644 index 000000000..0a0286496 --- /dev/null +++ b/app/workers/scheduler/follow_recommendations_scheduler.rb @@ -0,0 +1,61 @@ +# frozen_string_literal: true + +class Scheduler::FollowRecommendationsScheduler + include Sidekiq::Worker + include Redisable + + sidekiq_options retry: 0 + + # The maximum number of accounts that can be requested in one page from the + # API is 80, and the suggestions API does not allow pagination. This number + # leaves some room for accounts being filtered during live access + SET_SIZE = 100 + + def perform + # Maintaining a materialized view speeds-up subsequent queries significantly + AccountSummary.refresh + + fallback_recommendations = FollowRecommendation.safe.filtered.limit(SET_SIZE).index_by(&:account_id) + + I18n.available_locales.each do |locale| + recommendations = begin + if AccountSummary.safe.filtered.localized(locale).exists? # We can skip the work if no accounts with that language exist + FollowRecommendation.safe.filtered.localized(locale).limit(SET_SIZE).index_by(&:account_id) + else + {} + end + end + + # Use language-agnostic results if there are not enough language-specific ones + missing = SET_SIZE - recommendations.keys.size + + if missing.positive? + added = 0 + + # Avoid duplicate results + fallback_recommendations.each_value do |recommendation| + next if recommendations.key?(recommendation.account_id) + + recommendations[recommendation.account_id] = recommendation + added += 1 + + break if added >= missing + end + end + + redis.pipelined do + redis.del(key(locale)) + + recommendations.each_value do |recommendation| + redis.zadd(key(locale), recommendation.rank, recommendation.account_id) + end + end + end + end + + private + + def key(locale) + "follow_recommendations:#{locale}" + end +end diff --git a/config/locales/en.yml b/config/locales/en.yml index 3387b4df6..afab6d9b5 100644 --- a/config/locales/en.yml +++ b/config/locales/en.yml @@ -440,6 +440,14 @@ en: create: Add domain title: Block new e-mail domain title: Blocked e-mail domains + follow_recommendations: + description_html: "Follow recommendations help new users quickly find interesting content. When a user has not interacted with others enough to form personalized follow recommendations, these accounts are recommended instead. They are re-calculated on a daily basis from a mix of accounts with the highest recent engagements and highest local follower counts for a given language." + language: For language + status: Status + suppress: Suppress follow recommendation + suppressed: Suppressed + title: Follow recommendations + unsuppress: Restore follow recommendation instances: by_domain: Domain delivery_available: Delivery is available diff --git a/config/navigation.rb b/config/navigation.rb index 3a82c7971..b3462c48d 100644 --- a/config/navigation.rb +++ b/config/navigation.rb @@ -39,6 +39,7 @@ SimpleNavigation::Configuration.run do |navigation| s.item :accounts, safe_join([fa_icon('users fw'), t('admin.accounts.title')]), admin_accounts_url, highlights_on: %r{/admin/accounts|/admin/pending_accounts} s.item :invites, safe_join([fa_icon('user-plus fw'), t('admin.invites.title')]), admin_invites_path s.item :tags, safe_join([fa_icon('hashtag fw'), t('admin.tags.title')]), admin_tags_path, highlights_on: %r{/admin/tags} + s.item :follow_recommendations, safe_join([fa_icon('user-plus fw'), t('admin.follow_recommendations.title')]), admin_follow_recommendations_path, highlights_on: %r{/admin/follow_recommendations} s.item :instances, safe_join([fa_icon('cloud fw'), t('admin.instances.title')]), admin_instances_url(limited: whitelist_mode? ? nil : '1'), highlights_on: %r{/admin/instances|/admin/domain_blocks|/admin/domain_allows}, if: -> { current_user.admin? } s.item :email_domain_blocks, safe_join([fa_icon('envelope fw'), t('admin.email_domain_blocks.title')]), admin_email_domain_blocks_url, highlights_on: %r{/admin/email_domain_blocks}, if: -> { current_user.admin? } s.item :ip_blocks, safe_join([fa_icon('ban fw'), t('admin.ip_blocks.title')]), admin_ip_blocks_url, highlights_on: %r{/admin/ip_blocks}, if: -> { current_user.admin? } diff --git a/config/routes.rb b/config/routes.rb index eedd0de69..4661a7c11 100644 --- a/config/routes.rb +++ b/config/routes.rb @@ -292,6 +292,7 @@ Rails.application.routes.draw do end resources :account_moderation_notes, only: [:create, :destroy] + resource :follow_recommendations, only: [:show, :update] resources :tags, only: [:index, :show, :update] do collection do @@ -507,6 +508,7 @@ Rails.application.routes.draw do namespace :v2 do resources :media, only: [:create] get '/search', to: 'search#index', as: :search + resources :suggestions, only: [:index] end namespace :web do diff --git a/config/sidekiq.yml b/config/sidekiq.yml index 010923717..a8e4c7feb 100644 --- a/config/sidekiq.yml +++ b/config/sidekiq.yml @@ -25,6 +25,10 @@ cron: '<%= Random.rand(0..59) %> <%= Random.rand(0..2) %> * * *' class: Scheduler::FeedCleanupScheduler queue: scheduler + follow_recommendations_scheduler: + cron: '<%= Random.rand(0..59) %> <%= Random.rand(6..9) %> * * *' + class: Scheduler::FollowRecommendationsScheduler + queue: scheduler doorkeeper_cleanup_scheduler: cron: '<%= Random.rand(0..59) %> <%= Random.rand(0..2) %> * * 0' class: Scheduler::DoorkeeperCleanupScheduler diff --git a/db/migrate/20210322164601_create_account_summaries.rb b/db/migrate/20210322164601_create_account_summaries.rb new file mode 100644 index 000000000..b9faf180d --- /dev/null +++ b/db/migrate/20210322164601_create_account_summaries.rb @@ -0,0 +1,9 @@ +class CreateAccountSummaries < ActiveRecord::Migration[5.2] + def change + create_view :account_summaries, materialized: true + + # To be able to refresh the view concurrently, + # at least one unique index is required + safety_assured { add_index :account_summaries, :account_id, unique: true } + end +end diff --git a/db/migrate/20210323114347_create_follow_recommendations.rb b/db/migrate/20210323114347_create_follow_recommendations.rb new file mode 100644 index 000000000..77e729032 --- /dev/null +++ b/db/migrate/20210323114347_create_follow_recommendations.rb @@ -0,0 +1,5 @@ +class CreateFollowRecommendations < ActiveRecord::Migration[5.2] + def change + create_view :follow_recommendations + end +end diff --git a/db/migrate/20210324171613_create_follow_recommendation_suppressions.rb b/db/migrate/20210324171613_create_follow_recommendation_suppressions.rb new file mode 100644 index 000000000..c17a0be63 --- /dev/null +++ b/db/migrate/20210324171613_create_follow_recommendation_suppressions.rb @@ -0,0 +1,9 @@ +class CreateFollowRecommendationSuppressions < ActiveRecord::Migration[6.1] + def change + create_table :follow_recommendation_suppressions do |t| + t.references :account, null: false, foreign_key: { on_delete: :cascade }, index: { unique: true } + + t.timestamps + end + end +end diff --git a/db/schema.rb b/db/schema.rb index 4edaf5651..28f36abb1 100644 --- a/db/schema.rb +++ b/db/schema.rb @@ -2,15 +2,15 @@ # of editing this file, please use the migrations feature of Active Record to # incrementally modify your database, and then regenerate this schema definition. # -# Note that this schema.rb definition is the authoritative source for your -# database schema. If you need to create the application database on another -# system, you should be using db:schema:load, not running all the migrations -# from scratch. The latter is a flawed and unsustainable approach (the more migrations -# you'll amass, the slower it'll run and the greater likelihood for issues). +# This file is the source Rails uses to define your schema when running `bin/rails +# db:schema:load`. When creating a new database, `bin/rails db:schema:load` tends to +# be faster and is potentially less error prone than running all of your +# migrations from scratch. Old migrations may fail to apply correctly if those +# migrations use external dependencies or application code. # # It's strongly recommended that you check this file into your version control system. -ActiveRecord::Schema.define(version: 2021_03_08_133107) do +ActiveRecord::Schema.define(version: 2021_03_24_171613) do # These are extensions that must be enabled in order to support this database enable_extension "plpgsql" @@ -406,6 +406,13 @@ ActiveRecord::Schema.define(version: 2021_03_08_133107) do t.index ["tag_id"], name: "index_featured_tags_on_tag_id" end + create_table "follow_recommendation_suppressions", force: :cascade do |t| + t.bigint "account_id", null: false + t.datetime "created_at", precision: 6, null: false + t.datetime "updated_at", precision: 6, null: false + t.index ["account_id"], name: "index_follow_recommendation_suppressions_on_account_id", unique: true + end + create_table "follow_requests", force: :cascade do |t| t.datetime "created_at", null: false t.datetime "updated_at", null: false @@ -996,6 +1003,7 @@ ActiveRecord::Schema.define(version: 2021_03_08_133107) do add_foreign_key "favourites", "statuses", name: "fk_b0e856845e", on_delete: :cascade add_foreign_key "featured_tags", "accounts", on_delete: :cascade add_foreign_key "featured_tags", "tags", on_delete: :cascade + add_foreign_key "follow_recommendation_suppressions", "accounts", on_delete: :cascade add_foreign_key "follow_requests", "accounts", column: "target_account_id", name: "fk_9291ec025d", on_delete: :cascade add_foreign_key "follow_requests", "accounts", name: "fk_76d644b0e7", on_delete: :cascade add_foreign_key "follows", "accounts", column: "target_account_id", name: "fk_745ca29eac", on_delete: :cascade @@ -1079,4 +1087,47 @@ ActiveRecord::Schema.define(version: 2021_03_08_133107) do SQL add_index "instances", ["domain"], name: "index_instances_on_domain", unique: true + create_view "account_summaries", materialized: true, sql_definition: <<-SQL + SELECT accounts.id AS account_id, + mode() WITHIN GROUP (ORDER BY t0.language) AS language, + mode() WITHIN GROUP (ORDER BY t0.sensitive) AS sensitive + FROM (accounts + CROSS JOIN LATERAL ( SELECT statuses.account_id, + statuses.language, + statuses.sensitive + FROM statuses + WHERE ((statuses.account_id = accounts.id) AND (statuses.deleted_at IS NULL)) + ORDER BY statuses.id DESC + LIMIT 20) t0) + WHERE ((accounts.suspended_at IS NULL) AND (accounts.silenced_at IS NULL) AND (accounts.moved_to_account_id IS NULL) AND (accounts.discoverable = true) AND (accounts.locked = false)) + GROUP BY accounts.id; + SQL + add_index "account_summaries", ["account_id"], name: "index_account_summaries_on_account_id", unique: true + + create_view "follow_recommendations", sql_definition: <<-SQL + SELECT t0.account_id, + sum(t0.rank) AS rank, + array_agg(t0.reason) AS reason + FROM ( SELECT accounts.id AS account_id, + ((count(follows.id))::numeric / (1.0 + (count(follows.id))::numeric)) AS rank, + 'most_followed'::text AS reason + FROM ((follows + JOIN accounts ON ((accounts.id = follows.target_account_id))) + JOIN users ON ((users.account_id = follows.account_id))) + WHERE ((users.current_sign_in_at >= (now() - 'P30D'::interval)) AND (accounts.suspended_at IS NULL) AND (accounts.moved_to_account_id IS NULL) AND (accounts.silenced_at IS NULL) AND (accounts.locked = false) AND (accounts.discoverable = true)) + GROUP BY accounts.id + HAVING (count(follows.id) >= 5) + UNION ALL + SELECT accounts.id AS account_id, + (sum((status_stats.reblogs_count + status_stats.favourites_count)) / (1.0 + sum((status_stats.reblogs_count + status_stats.favourites_count)))) AS rank, + 'most_interactions'::text AS reason + FROM ((status_stats + JOIN statuses ON ((statuses.id = status_stats.status_id))) + JOIN accounts ON ((accounts.id = statuses.account_id))) + WHERE ((statuses.id >= (((date_part('epoch'::text, (now() - 'P30D'::interval)) * (1000)::double precision))::bigint << 16)) AND (accounts.suspended_at IS NULL) AND (accounts.moved_to_account_id IS NULL) AND (accounts.silenced_at IS NULL) AND (accounts.locked = false) AND (accounts.discoverable = true)) + GROUP BY accounts.id + HAVING (sum((status_stats.reblogs_count + status_stats.favourites_count)) >= (5)::numeric)) t0 + GROUP BY t0.account_id + ORDER BY (sum(t0.rank)) DESC; + SQL end diff --git a/db/views/account_summaries_v01.sql b/db/views/account_summaries_v01.sql new file mode 100644 index 000000000..5a632b622 --- /dev/null +++ b/db/views/account_summaries_v01.sql @@ -0,0 +1,22 @@ +SELECT + accounts.id AS account_id, + mode() WITHIN GROUP (ORDER BY language ASC) AS language, + mode() WITHIN GROUP (ORDER BY sensitive ASC) AS sensitive +FROM accounts +CROSS JOIN LATERAL ( + SELECT + statuses.account_id, + statuses.language, + statuses.sensitive + FROM statuses + WHERE statuses.account_id = accounts.id + AND statuses.deleted_at IS NULL + ORDER BY statuses.id DESC + LIMIT 20 +) t0 +WHERE accounts.suspended_at IS NULL + AND accounts.silenced_at IS NULL + AND accounts.moved_to_account_id IS NULL + AND accounts.discoverable = 't' + AND accounts.locked = 'f' +GROUP BY accounts.id diff --git a/db/views/follow_recommendations_v01.sql b/db/views/follow_recommendations_v01.sql new file mode 100644 index 000000000..799abeaee --- /dev/null +++ b/db/views/follow_recommendations_v01.sql @@ -0,0 +1,38 @@ +SELECT + account_id, + sum(rank) AS rank, + array_agg(reason) AS reason +FROM ( + SELECT + accounts.id AS account_id, + count(follows.id) / (1.0 + count(follows.id)) AS rank, + 'most_followed' AS reason + FROM follows + INNER JOIN accounts ON accounts.id = follows.target_account_id + INNER JOIN users ON users.account_id = follows.account_id + WHERE users.current_sign_in_at >= (now() - interval '30 days') + AND accounts.suspended_at IS NULL + AND accounts.moved_to_account_id IS NULL + AND accounts.silenced_at IS NULL + AND accounts.locked = 'f' + AND accounts.discoverable = 't' + GROUP BY accounts.id + HAVING count(follows.id) >= 5 + UNION ALL + SELECT accounts.id AS account_id, + sum(reblogs_count + favourites_count) / (1.0 + sum(reblogs_count + favourites_count)) AS rank, + 'most_interactions' AS reason + FROM status_stats + INNER JOIN statuses ON statuses.id = status_stats.status_id + INNER JOIN accounts ON accounts.id = statuses.account_id + WHERE statuses.id >= ((date_part('epoch', now() - interval '30 days') * 1000)::bigint << 16) + AND accounts.suspended_at IS NULL + AND accounts.moved_to_account_id IS NULL + AND accounts.silenced_at IS NULL + AND accounts.locked = 'f' + AND accounts.discoverable = 't' + GROUP BY accounts.id + HAVING sum(reblogs_count + favourites_count) >= 5 +) t0 +GROUP BY account_id +ORDER BY rank DESC diff --git a/spec/fabricators/follow_recommendation_suppression_fabricator.rb b/spec/fabricators/follow_recommendation_suppression_fabricator.rb new file mode 100644 index 000000000..4a6a07a66 --- /dev/null +++ b/spec/fabricators/follow_recommendation_suppression_fabricator.rb @@ -0,0 +1,3 @@ +Fabricator(:follow_recommendation_suppression) do + account +end diff --git a/spec/models/follow_recommendation_suppression_spec.rb b/spec/models/follow_recommendation_suppression_spec.rb new file mode 100644 index 000000000..39107a2b0 --- /dev/null +++ b/spec/models/follow_recommendation_suppression_spec.rb @@ -0,0 +1,4 @@ +require 'rails_helper' + +RSpec.describe FollowRecommendationSuppression, type: :model do +end -- cgit From ce2148c57111981be455a9953d3bb589cf53967f Mon Sep 17 00:00:00 2001 From: Eugen Rochko Date: Thu, 15 Apr 2021 05:00:25 +0200 Subject: Add `policy` param to `POST /api/v1/push/subscriptions` (#16040) With possible values `all`, `followed`, `follower`, and `none`, control from whom notifications will generate a Web Push alert --- .../api/v1/push/subscriptions_controller.rb | 28 +++---- .../api/web/push_subscriptions_controller.rb | 25 +++--- app/models/web/push_subscription.rb | 23 +++++- .../api/v1/push/subscriptions_controller_spec.rb | 28 ++++--- .../api/web/push_subscriptions_controller_spec.rb | 23 ++++-- spec/models/web/push_subscription_spec.rb | 94 ++++++++++++++++++++-- 6 files changed, 170 insertions(+), 51 deletions(-) (limited to 'app/controllers') diff --git a/app/controllers/api/v1/push/subscriptions_controller.rb b/app/controllers/api/v1/push/subscriptions_controller.rb index 0918c61e9..47f2e6440 100644 --- a/app/controllers/api/v1/push/subscriptions_controller.rb +++ b/app/controllers/api/v1/push/subscriptions_controller.rb @@ -3,13 +3,13 @@ class Api::V1::Push::SubscriptionsController < Api::BaseController before_action -> { doorkeeper_authorize! :push } before_action :require_user! - before_action :set_web_push_subscription - before_action :check_web_push_subscription, only: [:show, :update] + before_action :set_push_subscription + before_action :check_push_subscription, only: [:show, :update] def create - @web_subscription&.destroy! + @push_subscription&.destroy! - @web_subscription = ::Web::PushSubscription.create!( + @push_subscription = Web::PushSubscription.create!( endpoint: subscription_params[:endpoint], key_p256dh: subscription_params[:keys][:p256dh], key_auth: subscription_params[:keys][:auth], @@ -18,31 +18,31 @@ class Api::V1::Push::SubscriptionsController < Api::BaseController access_token_id: doorkeeper_token.id ) - render json: @web_subscription, serializer: REST::WebPushSubscriptionSerializer + render json: @push_subscription, serializer: REST::WebPushSubscriptionSerializer end def show - render json: @web_subscription, serializer: REST::WebPushSubscriptionSerializer + render json: @push_subscription, serializer: REST::WebPushSubscriptionSerializer end def update - @web_subscription.update!(data: data_params) - render json: @web_subscription, serializer: REST::WebPushSubscriptionSerializer + @push_subscription.update!(data: data_params) + render json: @push_subscription, serializer: REST::WebPushSubscriptionSerializer end def destroy - @web_subscription&.destroy! + @push_subscription&.destroy! render_empty end private - def set_web_push_subscription - @web_subscription = ::Web::PushSubscription.find_by(access_token_id: doorkeeper_token.id) + def set_push_subscription + @push_subscription = Web::PushSubscription.find_by(access_token_id: doorkeeper_token.id) end - def check_web_push_subscription - not_found if @web_subscription.nil? + def check_push_subscription + not_found if @push_subscription.nil? end def subscription_params @@ -52,6 +52,6 @@ class Api::V1::Push::SubscriptionsController < Api::BaseController def data_params return {} if params[:data].blank? - params.require(:data).permit(alerts: [:follow, :follow_request, :favourite, :reblog, :mention, :poll, :status]) + params.require(:data).permit(:policy, alerts: [:follow, :follow_request, :favourite, :reblog, :mention, :poll, :status]) end end diff --git a/app/controllers/api/web/push_subscriptions_controller.rb b/app/controllers/api/web/push_subscriptions_controller.rb index 1dce3e70f..bed57fc54 100644 --- a/app/controllers/api/web/push_subscriptions_controller.rb +++ b/app/controllers/api/web/push_subscriptions_controller.rb @@ -2,6 +2,7 @@ class Api::Web::PushSubscriptionsController < Api::Web::BaseController before_action :require_user! + before_action :set_push_subscription, only: :update def create active_session = current_session @@ -15,9 +16,11 @@ class Api::Web::PushSubscriptionsController < Api::Web::BaseController alerts_enabled = active_session.detection.device.mobile? || active_session.detection.device.tablet? data = { + policy: 'all', + alerts: { follow: alerts_enabled, - follow_request: false, + follow_request: alerts_enabled, favourite: alerts_enabled, reblog: alerts_enabled, mention: alerts_enabled, @@ -28,7 +31,7 @@ class Api::Web::PushSubscriptionsController < Api::Web::BaseController data.deep_merge!(data_params) if params[:data] - web_subscription = ::Web::PushSubscription.create!( + push_subscription = ::Web::PushSubscription.create!( endpoint: subscription_params[:endpoint], key_p256dh: subscription_params[:keys][:p256dh], key_auth: subscription_params[:keys][:auth], @@ -37,27 +40,27 @@ class Api::Web::PushSubscriptionsController < Api::Web::BaseController access_token_id: active_session.access_token_id ) - active_session.update!(web_push_subscription: web_subscription) + active_session.update!(web_push_subscription: push_subscription) - render json: web_subscription, serializer: REST::WebPushSubscriptionSerializer + render json: push_subscription, serializer: REST::WebPushSubscriptionSerializer end def update - params.require([:id]) - - web_subscription = ::Web::PushSubscription.find(params[:id]) - web_subscription.update!(data: data_params) - - render json: web_subscription, serializer: REST::WebPushSubscriptionSerializer + @push_subscription.update!(data: data_params) + render json: @push_subscription, serializer: REST::WebPushSubscriptionSerializer end private + def set_push_subscription + @push_subscription = ::Web::PushSubscription.find(params[:id]) + end + def subscription_params @subscription_params ||= params.require(:subscription).permit(:endpoint, keys: [:auth, :p256dh]) end def data_params - @data_params ||= params.require(:data).permit(alerts: [:follow, :follow_request, :favourite, :reblog, :mention, :poll, :status]) + @data_params ||= params.require(:data).permit(:policy, alerts: [:follow, :follow_request, :favourite, :reblog, :mention, :poll, :status]) end end diff --git a/app/models/web/push_subscription.rb b/app/models/web/push_subscription.rb index 7609b1bfc..6e46573ae 100644 --- a/app/models/web/push_subscription.rb +++ b/app/models/web/push_subscription.rb @@ -47,7 +47,7 @@ class Web::PushSubscription < ApplicationRecord end def pushable?(notification) - ActiveModel::Type::Boolean.new.cast(data&.dig('alerts', notification.type.to_s)) + policy_allows_notification?(notification) && alert_enabled_for_notification_type?(notification) end def associated_user @@ -100,4 +100,25 @@ class Web::PushSubscription < ApplicationRecord def contact_email @contact_email ||= ::Setting.site_contact_email end + + def alert_enabled_for_notification_type?(notification) + truthy?(data&.dig('alerts', notification.type.to_s)) + end + + def policy_allows_notification?(notification) + case data&.dig('policy') + when nil, 'all' + true + when 'none' + false + when 'followed' + notification.account.following?(notification.from_account) + when 'follower' + notification.from_account.following?(notification.account) + end + end + + def truthy?(val) + ActiveModel::Type::Boolean.new.cast(val) + end end diff --git a/spec/controllers/api/v1/push/subscriptions_controller_spec.rb b/spec/controllers/api/v1/push/subscriptions_controller_spec.rb index 01146294f..534d02879 100644 --- a/spec/controllers/api/v1/push/subscriptions_controller_spec.rb +++ b/spec/controllers/api/v1/push/subscriptions_controller_spec.rb @@ -27,20 +27,27 @@ describe Api::V1::Push::SubscriptionsController do let(:alerts_payload) do { data: { + policy: 'all', + alerts: { follow: true, + follow_request: true, favourite: false, reblog: true, mention: false, + poll: true, + status: false, } } }.with_indifferent_access end describe 'POST #create' do - it 'saves push subscriptions' do + before do post :create, params: create_payload + end + it 'saves push subscriptions' do push_subscription = Web::PushSubscription.find_by(endpoint: create_payload[:subscription][:endpoint]) expect(push_subscription.endpoint).to eq(create_payload[:subscription][:endpoint]) @@ -52,31 +59,34 @@ describe Api::V1::Push::SubscriptionsController do it 'replaces old subscription on repeat calls' do post :create, params: create_payload - post :create, params: create_payload - expect(Web::PushSubscription.where(endpoint: create_payload[:subscription][:endpoint]).count).to eq 1 end end describe 'PUT #update' do - it 'changes alert settings' do + before do post :create, params: create_payload put :update, params: alerts_payload + end + it 'changes alert settings' do push_subscription = Web::PushSubscription.find_by(endpoint: create_payload[:subscription][:endpoint]) - expect(push_subscription.data.dig('alerts', 'follow')).to eq(alerts_payload[:data][:alerts][:follow].to_s) - expect(push_subscription.data.dig('alerts', 'favourite')).to eq(alerts_payload[:data][:alerts][:favourite].to_s) - expect(push_subscription.data.dig('alerts', 'reblog')).to eq(alerts_payload[:data][:alerts][:reblog].to_s) - expect(push_subscription.data.dig('alerts', 'mention')).to eq(alerts_payload[:data][:alerts][:mention].to_s) + expect(push_subscription.data['policy']).to eq(alerts_payload[:data][:policy]) + + %w(follow follow_request favourite reblog mention poll status).each do |type| + expect(push_subscription.data['alerts'][type]).to eq(alerts_payload[:data][:alerts][type.to_sym].to_s) + end end end describe 'DELETE #destroy' do - it 'removes the subscription' do + before do post :create, params: create_payload delete :destroy + end + it 'removes the subscription' do expect(Web::PushSubscription.find_by(endpoint: create_payload[:subscription][:endpoint])).to be_nil end end diff --git a/spec/controllers/api/web/push_subscriptions_controller_spec.rb b/spec/controllers/api/web/push_subscriptions_controller_spec.rb index 381cdeab9..bda4a7661 100644 --- a/spec/controllers/api/web/push_subscriptions_controller_spec.rb +++ b/spec/controllers/api/web/push_subscriptions_controller_spec.rb @@ -22,11 +22,16 @@ describe Api::Web::PushSubscriptionsController do let(:alerts_payload) do { data: { + policy: 'all', + alerts: { follow: true, + follow_request: false, favourite: false, reblog: true, mention: false, + poll: true, + status: false, } } } @@ -59,10 +64,11 @@ describe Api::Web::PushSubscriptionsController do push_subscription = Web::PushSubscription.find_by(endpoint: create_payload[:subscription][:endpoint]) - expect(push_subscription.data['alerts']['follow']).to eq(alerts_payload[:data][:alerts][:follow].to_s) - expect(push_subscription.data['alerts']['favourite']).to eq(alerts_payload[:data][:alerts][:favourite].to_s) - expect(push_subscription.data['alerts']['reblog']).to eq(alerts_payload[:data][:alerts][:reblog].to_s) - expect(push_subscription.data['alerts']['mention']).to eq(alerts_payload[:data][:alerts][:mention].to_s) + expect(push_subscription.data['policy']).to eq 'all' + + %w(follow follow_request favourite reblog mention poll status).each do |type| + expect(push_subscription.data['alerts'][type]).to eq(alerts_payload[:data][:alerts][type.to_sym].to_s) + end end end end @@ -81,10 +87,11 @@ describe Api::Web::PushSubscriptionsController do push_subscription = Web::PushSubscription.find_by(endpoint: create_payload[:subscription][:endpoint]) - expect(push_subscription.data['alerts']['follow']).to eq(alerts_payload[:data][:alerts][:follow].to_s) - expect(push_subscription.data['alerts']['favourite']).to eq(alerts_payload[:data][:alerts][:favourite].to_s) - expect(push_subscription.data['alerts']['reblog']).to eq(alerts_payload[:data][:alerts][:reblog].to_s) - expect(push_subscription.data['alerts']['mention']).to eq(alerts_payload[:data][:alerts][:mention].to_s) + expect(push_subscription.data['policy']).to eq 'all' + + %w(follow follow_request favourite reblog mention poll status).each do |type| + expect(push_subscription.data['alerts'][type]).to eq(alerts_payload[:data][:alerts][type.to_sym].to_s) + end end end end diff --git a/spec/models/web/push_subscription_spec.rb b/spec/models/web/push_subscription_spec.rb index c6665611c..b44904369 100644 --- a/spec/models/web/push_subscription_spec.rb +++ b/spec/models/web/push_subscription_spec.rb @@ -1,16 +1,94 @@ require 'rails_helper' RSpec.describe Web::PushSubscription, type: :model do - let(:alerts) { { mention: true, reblog: false, follow: true, follow_request: false, favourite: true } } - let(:push_subscription) { Web::PushSubscription.new(data: { alerts: alerts }) } + let(:account) { Fabricate(:account) } + + let(:policy) { 'all' } + + let(:data) do + { + policy: policy, + + alerts: { + mention: true, + reblog: false, + follow: true, + follow_request: false, + favourite: true, + }, + } + end + + subject { described_class.new(data: data) } describe '#pushable?' do - it 'obeys alert settings' do - expect(push_subscription.send(:pushable?, Notification.new(activity_type: 'Mention'))).to eq true - expect(push_subscription.send(:pushable?, Notification.new(activity_type: 'Status'))).to eq false - expect(push_subscription.send(:pushable?, Notification.new(activity_type: 'Follow'))).to eq true - expect(push_subscription.send(:pushable?, Notification.new(activity_type: 'FollowRequest'))).to eq false - expect(push_subscription.send(:pushable?, Notification.new(activity_type: 'Favourite'))).to eq true + let(:notification_type) { :mention } + let(:notification) { Fabricate(:notification, account: account, type: notification_type) } + + %i(mention reblog follow follow_request favourite).each do |type| + context "when notification is a #{type}" do + let(:notification_type) { type } + + it "returns boolean corresonding to alert setting" do + expect(subject.pushable?(notification)).to eq data[:alerts][type] + end + end + end + + context 'when policy is all' do + let(:policy) { 'all' } + + it 'returns true' do + expect(subject.pushable?(notification)).to eq true + end + end + + context 'when policy is none' do + let(:policy) { 'none' } + + it 'returns false' do + expect(subject.pushable?(notification)).to eq false + end + end + + context 'when policy is followed' do + let(:policy) { 'followed' } + + context 'and notification is from someone you follow' do + before do + account.follow!(notification.from_account) + end + + it 'returns true' do + expect(subject.pushable?(notification)).to eq true + end + end + + context 'and notification is not from someone you follow' do + it 'returns false' do + expect(subject.pushable?(notification)).to eq false + end + end + end + + context 'when policy is follower' do + let(:policy) { 'follower' } + + context 'and notification is from someone who follows you' do + before do + notification.from_account.follow!(account) + end + + it 'returns true' do + expect(subject.pushable?(notification)).to eq true + end + end + + context 'and notification is not from someone who follows you' do + it 'returns false' do + expect(subject.pushable?(notification)).to eq false + end + end end end end -- cgit From daccc07dc170627b17564402296f6c8631d0cd97 Mon Sep 17 00:00:00 2001 From: Eugen Rochko Date: Sat, 24 Apr 2021 17:01:43 +0200 Subject: Change auto-following admin-selected accounts, show in recommendations (#16078) --- app/controllers/api/v1/suggestions_controller.rb | 10 ++-- app/lib/potential_friendship_tracker.rb | 10 ---- app/models/account_suggestions.rb | 25 +++++--- app/models/account_suggestions/global_source.rb | 37 ++++++++++++ .../past_interactions_source.rb | 36 ++++++++++++ app/models/account_suggestions/setting_source.rb | 68 ++++++++++++++++++++++ app/models/account_suggestions/source.rb | 34 +++++++++++ app/models/account_suggestions/suggestion.rb | 7 +++ app/models/follow_recommendation.rb | 15 ----- app/models/form/admin_settings.rb | 2 - app/services/bootstrap_timeline_service.rb | 37 +----------- app/validators/existing_username_validator.rb | 24 ++++++-- app/views/admin/settings/edit.html.haml | 5 +- config/locales/en.yml | 7 +-- config/settings.yml | 1 - spec/services/bootstrap_timeline_service_spec.rb | 38 ------------ 16 files changed, 228 insertions(+), 128 deletions(-) create mode 100644 app/models/account_suggestions/global_source.rb create mode 100644 app/models/account_suggestions/past_interactions_source.rb create mode 100644 app/models/account_suggestions/setting_source.rb create mode 100644 app/models/account_suggestions/source.rb create mode 100644 app/models/account_suggestions/suggestion.rb (limited to 'app/controllers') diff --git a/app/controllers/api/v1/suggestions_controller.rb b/app/controllers/api/v1/suggestions_controller.rb index b2788cc76..9737ae5cb 100644 --- a/app/controllers/api/v1/suggestions_controller.rb +++ b/app/controllers/api/v1/suggestions_controller.rb @@ -5,20 +5,20 @@ class Api::V1::SuggestionsController < Api::BaseController before_action -> { doorkeeper_authorize! :read } before_action :require_user! - before_action :set_accounts def index - render json: @accounts, each_serializer: REST::AccountSerializer + suggestions = suggestions_source.get(current_account, limit: limit_param(DEFAULT_ACCOUNTS_LIMIT)) + render json: suggestions.map(&:account), each_serializer: REST::AccountSerializer end def destroy - PotentialFriendshipTracker.remove(current_account.id, params[:id]) + suggestions_source.remove(current_account, params[:id]) render_empty end private - def set_accounts - @accounts = PotentialFriendshipTracker.get(current_account, limit_param(DEFAULT_ACCOUNTS_LIMIT)) + def suggestions_source + AccountSuggestions::PastInteractionsSource.new end end diff --git a/app/lib/potential_friendship_tracker.rb b/app/lib/potential_friendship_tracker.rb index e72d454b6..f5bc20346 100644 --- a/app/lib/potential_friendship_tracker.rb +++ b/app/lib/potential_friendship_tracker.rb @@ -27,15 +27,5 @@ class PotentialFriendshipTracker def remove(account_id, target_account_id) redis.zrem("interactions:#{account_id}", target_account_id) end - - def get(account, limit) - account_ids = redis.zrevrange("interactions:#{account.id}", 0, limit) - - return [] if account_ids.empty? || limit < 1 - - accounts = Account.searchable.where(id: account_ids).index_by(&:id) - - account_ids.map { |id| accounts[id.to_i] }.compact - end end end diff --git a/app/models/account_suggestions.rb b/app/models/account_suggestions.rb index 7fe9d618e..d1774e62f 100644 --- a/app/models/account_suggestions.rb +++ b/app/models/account_suggestions.rb @@ -1,17 +1,28 @@ # frozen_string_literal: true class AccountSuggestions - class Suggestion < ActiveModelSerializers::Model - attributes :account, :source - end + SOURCES = [ + AccountSuggestions::SettingSource, + AccountSuggestions::PastInteractionsSource, + AccountSuggestions::GlobalSource, + ].freeze def self.get(account, limit) - suggestions = PotentialFriendshipTracker.get(account, limit).map { |target_account| Suggestion.new(account: target_account, source: :past_interaction) } - suggestions.concat(FollowRecommendation.get(account, limit - suggestions.size, suggestions.map { |suggestion| suggestion.account.id }).map { |target_account| Suggestion.new(account: target_account, source: :global) }) if suggestions.size < limit - suggestions + SOURCES.each_with_object([]) do |source_class, suggestions| + source_suggestions = source_class.new.get( + account, + skip_account_ids: suggestions.map(&:account_id), + limit: limit - suggestions.size + ) + + suggestions.concat(source_suggestions) + end end def self.remove(account, target_account_id) - PotentialFriendshipTracker.remove(account.id, target_account_id) + SOURCES.each do |source_class| + source = source_class.new + source.remove(account, target_account_id) + end end end diff --git a/app/models/account_suggestions/global_source.rb b/app/models/account_suggestions/global_source.rb new file mode 100644 index 000000000..ac764de50 --- /dev/null +++ b/app/models/account_suggestions/global_source.rb @@ -0,0 +1,37 @@ +# frozen_string_literal: true + +class AccountSuggestions::GlobalSource < AccountSuggestions::Source + def key + :global + end + + def get(account, skip_account_ids: [], limit: 40) + account_ids = account_ids_for_locale(account.user_locale) - [account.id] - skip_account_ids + + as_ordered_suggestions( + scope(account).where(id: account_ids), + account_ids + ).take(limit) + end + + def remove(_account, _target_account_id) + nil + end + + private + + def scope(account) + Account.searchable + .followable_by(account) + .not_excluded_by_account(account) + .not_domain_blocked_by_account(account) + end + + def account_ids_for_locale(locale) + Redis.current.zrevrange("follow_recommendations:#{locale}", 0, -1).map(&:to_i) + end + + def to_ordered_list_key(account) + account.id + end +end diff --git a/app/models/account_suggestions/past_interactions_source.rb b/app/models/account_suggestions/past_interactions_source.rb new file mode 100644 index 000000000..d169394f1 --- /dev/null +++ b/app/models/account_suggestions/past_interactions_source.rb @@ -0,0 +1,36 @@ +# frozen_string_literal: true + +class AccountSuggestions::PastInteractionsSource < AccountSuggestions::Source + include Redisable + + def key + :past_interactions + end + + def get(account, skip_account_ids: [], limit: 40) + account_ids = account_ids_for_account(account.id, limit + skip_account_ids.size) - skip_account_ids + + as_ordered_suggestions( + scope.where(id: account_ids), + account_ids + ).take(limit) + end + + def remove(account, target_account_id) + redis.zrem("interactions:#{account.id}", target_account_id) + end + + private + + def scope + Account.searchable + end + + def account_ids_for_account(account_id, limit) + redis.zrevrange("interactions:#{account_id}", 0, limit).map(&:to_i) + end + + def to_ordered_list_key(account) + account.id + end +end diff --git a/app/models/account_suggestions/setting_source.rb b/app/models/account_suggestions/setting_source.rb new file mode 100644 index 000000000..be9eff233 --- /dev/null +++ b/app/models/account_suggestions/setting_source.rb @@ -0,0 +1,68 @@ +# frozen_string_literal: true + +class AccountSuggestions::SettingSource < AccountSuggestions::Source + def key + :staff + end + + def get(account, skip_account_ids: [], limit: 40) + return [] unless setting_enabled? + + as_ordered_suggestions( + scope(account).where(setting_to_where_condition).where.not(id: skip_account_ids), + usernames_and_domains + ).take(limit) + end + + def remove(_account, _target_account_id) + nil + end + + private + + def scope(account) + Account.searchable + .followable_by(account) + .not_excluded_by_account(account) + .not_domain_blocked_by_account(account) + .where(locked: false) + .where.not(id: account.id) + end + + def usernames_and_domains + @usernames_and_domains ||= setting_to_usernames_and_domains + end + + def setting_enabled? + setting.present? + end + + def setting_to_where_condition + usernames_and_domains.map do |(username, domain)| + Arel::Nodes::Grouping.new( + Account.arel_table[:username].lower.eq(username.downcase).and( + Account.arel_table[:domain].lower.eq(domain&.downcase) + ) + ) + end.reduce(:or) + end + + def setting_to_usernames_and_domains + setting.split(',').map do |str| + username, domain = str.strip.gsub(/\A@/, '').split('@', 2) + domain = nil if TagManager.instance.local_domain?(domain) + + next if username.blank? + + [username, domain] + end.compact + end + + def setting + Setting.bootstrap_timeline_accounts + end + + def to_ordered_list_key(account) + [account.username, account.domain] + end +end diff --git a/app/models/account_suggestions/source.rb b/app/models/account_suggestions/source.rb new file mode 100644 index 000000000..bd1068d20 --- /dev/null +++ b/app/models/account_suggestions/source.rb @@ -0,0 +1,34 @@ +# frozen_string_literal: true + +class AccountSuggestions::Source + def key + raise NotImplementedError + end + + def get(_account, **kwargs) + raise NotImplementedError + end + + def remove(_account, target_account_id) + raise NotImplementedError + end + + protected + + def as_ordered_suggestions(scope, ordered_list) + return [] if ordered_list.empty? + + map = scope.index_by(&method(:to_ordered_list_key)) + + ordered_list.map { |ordered_list_key| map[ordered_list_key] }.compact.map do |account| + AccountSuggestions::Suggestion.new( + account: account, + source: key + ) + end + end + + def to_ordered_list_key(_account) + raise NotImplementedError + end +end diff --git a/app/models/account_suggestions/suggestion.rb b/app/models/account_suggestions/suggestion.rb new file mode 100644 index 000000000..2c6f4d27f --- /dev/null +++ b/app/models/account_suggestions/suggestion.rb @@ -0,0 +1,7 @@ +# frozen_string_literal: true + +class AccountSuggestions::Suggestion < ActiveModelSerializers::Model + attributes :account, :source + + delegate :id, to: :account, prefix: true +end diff --git a/app/models/follow_recommendation.rb b/app/models/follow_recommendation.rb index c4355224d..6670b6560 100644 --- a/app/models/follow_recommendation.rb +++ b/app/models/follow_recommendation.rb @@ -21,19 +21,4 @@ class FollowRecommendation < ApplicationRecord def readonly? true end - - def self.get(account, limit, exclude_account_ids = []) - account_ids = Redis.current.zrevrange("follow_recommendations:#{account.user_locale}", 0, -1).map(&:to_i) - exclude_account_ids - [account.id] - - return [] if account_ids.empty? || limit < 1 - - accounts = Account.followable_by(account) - .not_excluded_by_account(account) - .not_domain_blocked_by_account(account) - .where(id: account_ids) - .limit(limit) - .index_by(&:id) - - account_ids.map { |id| accounts[id] }.compact - end end diff --git a/app/models/form/admin_settings.rb b/app/models/form/admin_settings.rb index b5c3dcdbe..6fc7c56fd 100644 --- a/app/models/form/admin_settings.rb +++ b/app/models/form/admin_settings.rb @@ -16,7 +16,6 @@ class Form::AdminSettings open_deletion timeline_preview show_staff_badge - enable_bootstrap_timeline_accounts bootstrap_timeline_accounts theme min_invite_role @@ -41,7 +40,6 @@ class Form::AdminSettings open_deletion timeline_preview show_staff_badge - enable_bootstrap_timeline_accounts activity_api_enabled peers_api_enabled show_known_fediverse_at_about_page diff --git a/app/services/bootstrap_timeline_service.rb b/app/services/bootstrap_timeline_service.rb index 8412aa7e7..e1a1b98c3 100644 --- a/app/services/bootstrap_timeline_service.rb +++ b/app/services/bootstrap_timeline_service.rb @@ -5,48 +5,13 @@ class BootstrapTimelineService < BaseService @source_account = source_account autofollow_inviter! - autofollow_bootstrap_timeline_accounts! if Setting.enable_bootstrap_timeline_accounts end private def autofollow_inviter! return unless @source_account&.user&.invite&.autofollow? - FollowService.new.call(@source_account, @source_account.user.invite.user.account) - end - - def autofollow_bootstrap_timeline_accounts! - bootstrap_timeline_accounts.each do |target_account| - begin - FollowService.new.call(@source_account, target_account) - rescue ActiveRecord::RecordNotFound, Mastodon::NotPermittedError - nil - end - end - end - - def bootstrap_timeline_accounts - return @bootstrap_timeline_accounts if defined?(@bootstrap_timeline_accounts) - - @bootstrap_timeline_accounts = bootstrap_timeline_accounts_usernames.empty? ? admin_accounts : local_unlocked_accounts(bootstrap_timeline_accounts_usernames) - end - - def bootstrap_timeline_accounts_usernames - @bootstrap_timeline_accounts_usernames ||= (Setting.bootstrap_timeline_accounts || '').split(',').map { |str| str.strip.gsub(/\A@/, '') }.reject(&:blank?) - end - def admin_accounts - User.admins - .includes(:account) - .where(accounts: { locked: false }) - .map(&:account) - end - - def local_unlocked_accounts(usernames) - Account.local - .without_suspended - .where(username: usernames) - .where(locked: false) - .where(moved_to_account_id: nil) + FollowService.new.call(@source_account, @source_account.user.invite.user.account) end end diff --git a/app/validators/existing_username_validator.rb b/app/validators/existing_username_validator.rb index 723302ec9..afbe0c635 100644 --- a/app/validators/existing_username_validator.rb +++ b/app/validators/existing_username_validator.rb @@ -4,11 +4,25 @@ class ExistingUsernameValidator < ActiveModel::EachValidator def validate_each(record, attribute, value) return if value.blank? - if options[:multiple] - missing_usernames = value.split(',').map { |username| username.strip.gsub(/\A@/, '') }.filter_map { |username| username unless Account.find_local(username) } - record.errors.add(attribute, I18n.t('existing_username_validator.not_found_multiple', usernames: missing_usernames.join(', '))) if missing_usernames.any? - else - record.errors.add(attribute, I18n.t('existing_username_validator.not_found')) unless Account.find_local(value.strip.gsub(/\A@/, '')) + usernames_and_domains = begin + value.split(',').map do |str| + username, domain = str.strip.gsub(/\A@/, '').split('@') + domain = nil if TagManager.instance.local_domain?(domain) + + next if username.blank? + + [str, username, domain] + end.compact + end + + usernames_with_no_accounts = usernames_and_domains.filter_map do |(str, username, domain)| + str unless Account.find_remote(username, domain) + end + + if usernames_with_no_accounts.any? && options[:multiple] + record.errors.add(attribute, I18n.t('existing_username_validator.not_found_multiple', usernames: usernames_with_no_accounts.join(', '))) + elsif usernames_with_no_accounts.any? || usernames_and_domains.size > 1 + record.errors.add(attribute, I18n.t('existing_username_validator.not_found')) end end end diff --git a/app/views/admin/settings/edit.html.haml b/app/views/admin/settings/edit.html.haml index 7783dbfeb..33bfc43d3 100644 --- a/app/views/admin/settings/edit.html.haml +++ b/app/views/admin/settings/edit.html.haml @@ -50,10 +50,7 @@ %hr.spacer/ .fields-group - = f.input :enable_bootstrap_timeline_accounts, as: :boolean, wrapper: :with_label, label: t('admin.settings.enable_bootstrap_timeline_accounts.title'), hint: t('admin.settings.enable_bootstrap_timeline_accounts.desc_html') - - .fields-group - = f.input :bootstrap_timeline_accounts, wrapper: :with_block_label, label: t('admin.settings.bootstrap_timeline_accounts.title'), hint: t('admin.settings.bootstrap_timeline_accounts.desc_html'), disabled: !Setting.enable_bootstrap_timeline_accounts + = f.input :bootstrap_timeline_accounts, wrapper: :with_block_label, label: t('admin.settings.bootstrap_timeline_accounts.title'), hint: t('admin.settings.bootstrap_timeline_accounts.desc_html') %hr.spacer/ diff --git a/config/locales/en.yml b/config/locales/en.yml index 765f71250..1b41ee063 100644 --- a/config/locales/en.yml +++ b/config/locales/en.yml @@ -564,8 +564,8 @@ en: desc_html: Counts of locally published posts, active users, and new registrations in weekly buckets title: Publish aggregate statistics about user activity in the API bootstrap_timeline_accounts: - desc_html: Separate multiple usernames by comma. Only local and unlocked accounts will work. Default when empty is all local admins. - title: Default follows for new users + desc_html: Separate multiple usernames by comma. These accounts will be guaranteed to be shown in follow recommendations + title: Recommend these accounts to new users contact_information: email: Business e-mail username: Contact username @@ -582,9 +582,6 @@ en: users: To logged-in local users domain_blocks_rationale: title: Show rationale - enable_bootstrap_timeline_accounts: - desc_html: Make new users automatically follow configured accounts so their home feed doesn't start out empty - title: Enable default follows for new users hero: desc_html: Displayed on the frontpage. At least 600x100px recommended. When not set, falls back to server thumbnail title: Hero image diff --git a/config/settings.yml b/config/settings.yml index b79ea620c..06cee2532 100644 --- a/config/settings.yml +++ b/config/settings.yml @@ -62,7 +62,6 @@ defaults: &defaults - mod - moderator disallowed_hashtags: # space separated string or list of hashtags without the hash - enable_bootstrap_timeline_accounts: true bootstrap_timeline_accounts: '' activity_api_enabled: true peers_api_enabled: true diff --git a/spec/services/bootstrap_timeline_service_spec.rb b/spec/services/bootstrap_timeline_service_spec.rb index a28d2407c..880ca4f0d 100644 --- a/spec/services/bootstrap_timeline_service_spec.rb +++ b/spec/services/bootstrap_timeline_service_spec.rb @@ -1,42 +1,4 @@ require 'rails_helper' RSpec.describe BootstrapTimelineService, type: :service do - subject { described_class.new } - - describe '#call' do - let(:source_account) { Fabricate(:account) } - - context 'when setting is empty' do - let!(:admin) { Fabricate(:user, admin: true) } - - before do - Setting.bootstrap_timeline_accounts = nil - subject.call(source_account) - end - - it 'follows admin accounts from account' do - expect(source_account.following?(admin.account)).to be true - end - end - - context 'when setting is set' do - let!(:alice) { Fabricate(:account, username: 'alice') } - let!(:bob) { Fabricate(:account, username: 'bob') } - let!(:eve) { Fabricate(:account, username: 'eve', suspended: true) } - - before do - Setting.bootstrap_timeline_accounts = 'alice, @bob, eve, unknown' - subject.call(source_account) - end - - it 'follows found accounts from account' do - expect(source_account.following?(alice)).to be true - expect(source_account.following?(bob)).to be true - end - - it 'does not follow suspended account' do - expect(source_account.following?(eve)).to be false - end - end - end end -- cgit From 1f4751102364a3225233ad7d97a68e165c30ba64 Mon Sep 17 00:00:00 2001 From: abcang Date: Sun, 25 Apr 2021 13:34:48 +0900 Subject: Improve media attached status query (#16105) --- app/controllers/accounts_controller.rb | 6 +----- app/controllers/admin/statuses_controller.rb | 3 +-- app/controllers/api/v1/accounts/statuses_controller.rb | 2 +- 3 files changed, 3 insertions(+), 8 deletions(-) (limited to 'app/controllers') diff --git a/app/controllers/accounts_controller.rb b/app/controllers/accounts_controller.rb index 7753ebccc..f15961552 100644 --- a/app/controllers/accounts_controller.rb +++ b/app/controllers/accounts_controller.rb @@ -77,11 +77,7 @@ class AccountsController < ApplicationController end def only_media_scope - Status.where(id: account_media_status_ids) - end - - def account_media_status_ids - @account.media_attachments.attached.reorder(nil).select(:status_id).group(:status_id) + Status.joins(:media_attachments).group(:id) end def no_replies_scope diff --git a/app/controllers/admin/statuses_controller.rb b/app/controllers/admin/statuses_controller.rb index d7c192f0d..df1bd1bc7 100644 --- a/app/controllers/admin/statuses_controller.rb +++ b/app/controllers/admin/statuses_controller.rb @@ -14,8 +14,7 @@ module Admin @statuses = @account.statuses.where(visibility: [:public, :unlisted]) if params[:media] - account_media_status_ids = @account.media_attachments.attached.reorder(nil).select(:status_id).group(:status_id) - @statuses.merge!(Status.where(id: account_media_status_ids)) + @statuses.merge!(Status.joins(:media_attachments).group(:id)) end @statuses = @statuses.preload(:media_attachments, :mentions).page(params[:page]).per(PER_PAGE) diff --git a/app/controllers/api/v1/accounts/statuses_controller.rb b/app/controllers/api/v1/accounts/statuses_controller.rb index 92ccb8061..b1e478f8d 100644 --- a/app/controllers/api/v1/accounts/statuses_controller.rb +++ b/app/controllers/api/v1/accounts/statuses_controller.rb @@ -42,7 +42,7 @@ class Api::V1::Accounts::StatusesController < Api::BaseController end def only_media_scope - Status.joins(:media_attachments).merge(@account.media_attachments.reorder(nil)).group(:id) + Status.joins(:media_attachments).group(:id) end def pinned_scope -- cgit From d0fc69d7213cab03c57947cac94bad1fd9cc6dba Mon Sep 17 00:00:00 2001 From: abcang Date: Tue, 27 Apr 2021 01:57:46 +0900 Subject: Further improve the media attached status query for accounts (#16106) --- app/controllers/accounts_controller.rb | 2 +- app/controllers/admin/statuses_controller.rb | 2 +- app/controllers/api/v1/accounts/statuses_controller.rb | 2 +- ...2_add_index_on_media_attachments_account_id_status_id.rb | 13 +++++++++++++ db/schema.rb | 4 ++-- 5 files changed, 18 insertions(+), 5 deletions(-) create mode 100644 db/migrate/20210425135952_add_index_on_media_attachments_account_id_status_id.rb (limited to 'app/controllers') diff --git a/app/controllers/accounts_controller.rb b/app/controllers/accounts_controller.rb index f15961552..8210918d8 100644 --- a/app/controllers/accounts_controller.rb +++ b/app/controllers/accounts_controller.rb @@ -77,7 +77,7 @@ class AccountsController < ApplicationController end def only_media_scope - Status.joins(:media_attachments).group(:id) + Status.joins(:media_attachments).merge(@account.media_attachments.reorder(nil)).group(:id) end def no_replies_scope diff --git a/app/controllers/admin/statuses_controller.rb b/app/controllers/admin/statuses_controller.rb index df1bd1bc7..ef279509d 100644 --- a/app/controllers/admin/statuses_controller.rb +++ b/app/controllers/admin/statuses_controller.rb @@ -14,7 +14,7 @@ module Admin @statuses = @account.statuses.where(visibility: [:public, :unlisted]) if params[:media] - @statuses.merge!(Status.joins(:media_attachments).group(:id)) + @statuses.merge!(Status.joins(:media_attachments).merge(@account.media_attachments.reorder(nil)).group(:id)) end @statuses = @statuses.preload(:media_attachments, :mentions).page(params[:page]).per(PER_PAGE) diff --git a/app/controllers/api/v1/accounts/statuses_controller.rb b/app/controllers/api/v1/accounts/statuses_controller.rb index b1e478f8d..92ccb8061 100644 --- a/app/controllers/api/v1/accounts/statuses_controller.rb +++ b/app/controllers/api/v1/accounts/statuses_controller.rb @@ -42,7 +42,7 @@ class Api::V1::Accounts::StatusesController < Api::BaseController end def only_media_scope - Status.joins(:media_attachments).group(:id) + Status.joins(:media_attachments).merge(@account.media_attachments.reorder(nil)).group(:id) end def pinned_scope diff --git a/db/migrate/20210425135952_add_index_on_media_attachments_account_id_status_id.rb b/db/migrate/20210425135952_add_index_on_media_attachments_account_id_status_id.rb new file mode 100644 index 000000000..5ef2d3c39 --- /dev/null +++ b/db/migrate/20210425135952_add_index_on_media_attachments_account_id_status_id.rb @@ -0,0 +1,13 @@ +class AddIndexOnMediaAttachmentsAccountIdStatusId < ActiveRecord::Migration[5.2] + disable_ddl_transaction! + + def up + add_index :media_attachments, [:account_id, :status_id], order: { status_id: :desc }, algorithm: :concurrently + remove_index :media_attachments, :account_id, algorithm: :concurrently + end + + def down + add_index :media_attachments, :account_id, algorithm: :concurrently + remove_index :media_attachments, [:account_id, :status_id], order: { status_id: :desc }, algorithm: :concurrently + end +end diff --git a/db/schema.rb b/db/schema.rb index 8dc0661fc..0d951ee95 100644 --- a/db/schema.rb +++ b/db/schema.rb @@ -10,7 +10,7 @@ # # It's strongly recommended that you check this file into your version control system. -ActiveRecord::Schema.define(version: 2021_04_21_121431) do +ActiveRecord::Schema.define(version: 2021_04_25_135952) do # These are extensions that must be enabled in order to support this database enable_extension "plpgsql" @@ -541,7 +541,7 @@ ActiveRecord::Schema.define(version: 2021_04_21_121431) do t.integer "thumbnail_file_size" t.datetime "thumbnail_updated_at" t.string "thumbnail_remote_url" - t.index ["account_id"], name: "index_media_attachments_on_account_id" + t.index ["account_id", "status_id"], name: "index_media_attachments_on_account_id_and_status_id", order: { status_id: :desc } t.index ["scheduled_status_id"], name: "index_media_attachments_on_scheduled_status_id" t.index ["shortcode"], name: "index_media_attachments_on_shortcode", unique: true t.index ["status_id"], name: "index_media_attachments_on_status_id" -- cgit From 8c44b723bb7505eb5923019b454ce9abd7ea59c0 Mon Sep 17 00:00:00 2001 From: Claire Date: Mon, 3 May 2021 15:45:19 +0200 Subject: Change confirmations controller to redirect to / for approved users (#16151) Clicking the confirmation link multiple times currently leads to entering account settings, which can be confusing. This commit changes that so that it redirects to the root path, so it behaves the same way as clicking only once in most cases. --- app/controllers/auth/confirmations_controller.rb | 4 +- .../auth/confirmations_controller_spec.rb | 46 ++++++++++++++++++++++ 2 files changed, 49 insertions(+), 1 deletion(-) (limited to 'app/controllers') diff --git a/app/controllers/auth/confirmations_controller.rb b/app/controllers/auth/confirmations_controller.rb index 898525269..1475bbcef 100644 --- a/app/controllers/auth/confirmations_controller.rb +++ b/app/controllers/auth/confirmations_controller.rb @@ -17,7 +17,9 @@ class Auth::ConfirmationsController < Devise::ConfirmationsController private def require_unconfirmed! - redirect_to edit_user_registration_path if user_signed_in? && current_user.confirmed? && current_user.unconfirmed_email.blank? + if user_signed_in? && current_user.confirmed? && current_user.unconfirmed_email.blank? + redirect_to(current_user.approved? ? root_path : edit_user_registration_path) + end end def set_body_classes diff --git a/spec/controllers/auth/confirmations_controller_spec.rb b/spec/controllers/auth/confirmations_controller_spec.rb index 0b6b74ff9..8469119d2 100644 --- a/spec/controllers/auth/confirmations_controller_spec.rb +++ b/spec/controllers/auth/confirmations_controller_spec.rb @@ -32,6 +32,52 @@ describe Auth::ConfirmationsController, type: :controller do end end + context 'when user is unconfirmed and unapproved' do + let!(:user) { Fabricate(:user, confirmation_token: 'foobar', confirmed_at: nil, approved: false) } + + before do + allow(BootstrapTimelineWorker).to receive(:perform_async) + @request.env['devise.mapping'] = Devise.mappings[:user] + get :show, params: { confirmation_token: 'foobar' } + end + + it 'redirects to login' do + expect(response).to redirect_to(new_user_session_path) + end + end + + context 'when user is already confirmed' do + let!(:user) { Fabricate(:user) } + + before do + allow(BootstrapTimelineWorker).to receive(:perform_async) + @request.env['devise.mapping'] = Devise.mappings[:user] + sign_in(user, scope: :user) + get :show, params: { confirmation_token: 'foobar' } + end + + it 'redirects to root path' do + expect(response).to redirect_to(root_path) + end + end + + context 'when user is already confirmed but unapproved' do + let!(:user) { Fabricate(:user, approved: false) } + + before do + allow(BootstrapTimelineWorker).to receive(:perform_async) + @request.env['devise.mapping'] = Devise.mappings[:user] + user.approved = false + user.save! + sign_in(user, scope: :user) + get :show, params: { confirmation_token: 'foobar' } + end + + it 'redirects to settings' do + expect(response).to redirect_to(edit_user_registration_path) + end + end + context 'when user is updating email' do let!(:user) { Fabricate(:user, confirmation_token: 'foobar', unconfirmed_email: 'new-email@example.com') } -- cgit From 059df83d1dcabb27d2e638b4835791bd570ec779 Mon Sep 17 00:00:00 2001 From: Eugen Rochko Date: Wed, 5 May 2021 19:44:35 +0200 Subject: Fix database serialization failure returning HTTP 500 (#16101) Database serialization failure occurs when a read-replica is used and a query takes long enough that rows on the primary database become unavailable. It should return HTTP 503 as it is temporary. Re-order rescue definitions according to their status codes --- app/controllers/application_controller.rb | 15 +++++++-------- 1 file changed, 7 insertions(+), 8 deletions(-) (limited to 'app/controllers') diff --git a/app/controllers/application_controller.rb b/app/controllers/application_controller.rb index 6361d4b27..b4fb83661 100644 --- a/app/controllers/application_controller.rb +++ b/app/controllers/application_controller.rb @@ -18,17 +18,16 @@ class ApplicationController < ActionController::Base helper_method :use_seamless_external_login? helper_method :whitelist_mode? - rescue_from ActionController::RoutingError, with: :not_found - rescue_from ActionController::InvalidAuthenticityToken, with: :unprocessable_entity - rescue_from ActionController::UnknownFormat, with: :not_acceptable - rescue_from ActionController::ParameterMissing, with: :bad_request - rescue_from Paperclip::AdapterRegistry::NoHandlerError, with: :bad_request - rescue_from ActiveRecord::RecordNotFound, with: :not_found + rescue_from ActionController::ParameterMissing, Paperclip::AdapterRegistry::NoHandlerError, with: :bad_request rescue_from Mastodon::NotPermittedError, with: :forbidden - rescue_from HTTP::Error, OpenSSL::SSL::SSLError, with: :internal_server_error - rescue_from Mastodon::RaceConditionError, Seahorse::Client::NetworkingError, Stoplight::Error::RedLight, with: :service_unavailable + rescue_from ActionController::RoutingError, ActiveRecord::RecordNotFound, with: :not_found + rescue_from ActionController::UnknownFormat, with: :not_acceptable + rescue_from ActionController::InvalidAuthenticityToken, with: :unprocessable_entity rescue_from Mastodon::RateLimitExceededError, with: :too_many_requests + rescue_from HTTP::Error, OpenSSL::SSL::SSLError, with: :internal_server_error + rescue_from Mastodon::RaceConditionError, Seahorse::Client::NetworkingError, Stoplight::Error::RedLight, ActiveRecord::SerializationFailure, with: :service_unavailable + before_action :store_current_location, except: :raise_not_found, unless: :devise_controller? before_action :require_functional!, if: :user_signed_in? -- cgit From 351c74459084ccffce1333b57c2af9a6b55cac8d Mon Sep 17 00:00:00 2001 From: Eugen Rochko Date: Wed, 5 May 2021 21:16:55 +0200 Subject: Fix error when trying to render component for media without meta (#16112) --- app/controllers/statuses_controller.rb | 5 -- app/helpers/statuses_helper.rb | 80 +++++++++++++++++++++++++++ app/views/accounts/_header.html.haml | 4 +- app/views/media/player.html.haml | 7 ++- app/views/statuses/_detailed_status.html.haml | 24 +++----- app/views/statuses/_poll.html.haml | 4 +- app/views/statuses/_simple_status.html.haml | 24 +++----- app/views/statuses/_status.html.haml | 7 +-- app/views/statuses/embed.html.haml | 2 +- 9 files changed, 111 insertions(+), 46 deletions(-) (limited to 'app/controllers') diff --git a/app/controllers/statuses_controller.rb b/app/controllers/statuses_controller.rb index 87612a296..c52170d08 100644 --- a/app/controllers/statuses_controller.rb +++ b/app/controllers/statuses_controller.rb @@ -16,7 +16,6 @@ class StatusesController < ApplicationController before_action :set_referrer_policy_header, only: :show before_action :set_cache_headers before_action :set_body_classes - before_action :set_autoplay, only: :embed skip_around_action :set_locale, if: -> { request.format == :json } skip_before_action :require_functional!, only: [:show, :embed], unless: :whitelist_mode? @@ -82,8 +81,4 @@ class StatusesController < ApplicationController def set_referrer_policy_header response.headers['Referrer-Policy'] = 'origin' unless @status.distributable? end - - def set_autoplay - @autoplay = truthy_param?(:autoplay) - end end diff --git a/app/helpers/statuses_helper.rb b/app/helpers/statuses_helper.rb index 1f654f34f..25f079e9d 100644 --- a/app/helpers/statuses_helper.rb +++ b/app/helpers/statuses_helper.rb @@ -130,4 +130,84 @@ module StatusesHelper def embedded_view? params[:controller] == EMBEDDED_CONTROLLER && params[:action] == EMBEDDED_ACTION end + + def render_video_component(status, **options) + video = status.media_attachments.first + + meta = video.file.meta || {} + + component_params = { + sensitive: sensitized?(status, current_account), + src: full_asset_url(video.file.url(:original)), + preview: full_asset_url(video.thumbnail.present? ? video.thumbnail.url : video.file.url(:small)), + alt: video.description, + blurhash: video.blurhash, + frameRate: meta.dig('original', 'frame_rate'), + inline: true, + media: [ + ActiveModelSerializers::SerializableResource.new(video, serializer: REST::MediaAttachmentSerializer), + ].as_json, + }.merge(**options) + + react_component :video, component_params do + render partial: 'statuses/attachment_list', locals: { attachments: status.media_attachments } + end + end + + def render_audio_component(status, **options) + audio = status.media_attachments.first + + meta = audio.file.meta || {} + + component_params = { + src: full_asset_url(audio.file.url(:original)), + poster: full_asset_url(audio.thumbnail.present? ? audio.thumbnail.url : status.account.avatar_static_url), + alt: audio.description, + backgroundColor: meta.dig('colors', 'background'), + foregroundColor: meta.dig('colors', 'foreground'), + accentColor: meta.dig('colors', 'accent'), + duration: meta.dig('original', 'duration'), + }.merge(**options) + + react_component :audio, component_params do + render partial: 'statuses/attachment_list', locals: { attachments: status.media_attachments } + end + end + + def render_media_gallery_component(status, **options) + component_params = { + sensitive: sensitized?(status, current_account), + autoplay: prefers_autoplay?, + media: status.media_attachments.map { |a| ActiveModelSerializers::SerializableResource.new(a, serializer: REST::MediaAttachmentSerializer).as_json }, + }.merge(**options) + + react_component :media_gallery, component_params do + render partial: 'statuses/attachment_list', locals: { attachments: status.media_attachments } + end + end + + def render_card_component(status, **options) + component_params = { + sensitive: sensitized?(status, current_account), + maxDescription: 160, + card: ActiveModelSerializers::SerializableResource.new(status.preview_card, serializer: REST::PreviewCardSerializer).as_json, + }.merge(**options) + + react_component :card, component_params + end + + def render_poll_component(status, **options) + component_params = { + disabled: true, + poll: ActiveModelSerializers::SerializableResource.new(status.preloadable_poll, serializer: REST::PollSerializer, scope: current_user, scope_name: :current_user).as_json, + }.merge(**options) + + react_component :poll, component_params do + render partial: 'statuses/poll', locals: { status: status, poll: status.preloadable_poll, autoplay: prefers_autoplay? } + end + end + + def prefers_autoplay? + ActiveModel::Type::Boolean.new.cast(params[:autoplay]) || current_user&.setting_auto_play_gif + end end diff --git a/app/views/accounts/_header.html.haml b/app/views/accounts/_header.html.haml index 4ef9f9478..cae5a5ac9 100644 --- a/app/views/accounts/_header.html.haml +++ b/app/views/accounts/_header.html.haml @@ -1,9 +1,9 @@ .public-account-header{:class => ("inactive" if account.moved?)} .public-account-header__image - = image_tag (current_account&.user&.setting_auto_play_gif ? account.header_original_url : account.header_static_url), class: 'parallax' + = image_tag (prefers_autoplay? ? account.header_original_url : account.header_static_url), class: 'parallax' .public-account-header__bar = link_to short_account_url(account), class: 'avatar' do - = image_tag (current_account&.user&.setting_auto_play_gif ? account.avatar_original_url : account.avatar_static_url), id: 'profile_page_avatar', data: {original: full_asset_url(account.avatar_original_url), static: full_asset_url(account.avatar_static_url), autoplay: current_account&.user&.setting_auto_play_gif} + = image_tag (prefers_autoplay? ? account.avatar_original_url : account.avatar_static_url), id: 'profile_page_avatar', data: { original: full_asset_url(account.avatar_original_url), static: full_asset_url(account.avatar_static_url), autoplay: prefers_autoplay? } .public-account-header__tabs .public-account-header__tabs__name %h1 diff --git a/app/views/media/player.html.haml b/app/views/media/player.html.haml index 95e37bb22..f00c8f040 100644 --- a/app/views/media/player.html.haml +++ b/app/views/media/player.html.haml @@ -2,8 +2,11 @@ = render_initial_state = javascript_pack_tag 'public', crossorigin: 'anonymous' +:ruby + meta = @media_attachment.file.meta || {} + - if @media_attachment.video? - = react_component :video, src: @media_attachment.file.url(:original), preview: @media_attachment.thumbnail.present? ? @media_attachment.thumbnail.url : @media_attachment.file.url(:small), frameRate: @media_attachment.file.meta.dig('original', 'frame_rate'), blurhash: @media_attachment.blurhash, width: 670, height: 380, editable: true, detailed: true, inline: true, alt: @media_attachment.description, media: [ActiveModelSerializers::SerializableResource.new(@media_attachment, serializer: REST::MediaAttachmentSerializer)].as_json do + = react_component :video, src: @media_attachment.file.url(:original), preview: @media_attachment.thumbnail.present? ? @media_attachment.thumbnail.url : @media_attachment.file.url(:small), frameRate: meta.dig('original', 'frame_rate'), blurhash: @media_attachment.blurhash, width: 670, height: 380, editable: true, detailed: true, inline: true, alt: @media_attachment.description, media: [ActiveModelSerializers::SerializableResource.new(@media_attachment, serializer: REST::MediaAttachmentSerializer)].as_json do %video{ controls: 'controls' } %source{ src: @media_attachment.file.url(:original) } - elsif @media_attachment.gifv? @@ -11,6 +14,6 @@ %video{ autoplay: 'autoplay', muted: 'muted', loop: 'loop' } %source{ src: @media_attachment.file.url(:original) } - elsif @media_attachment.audio? - = react_component :audio, src: @media_attachment.file.url(:original), poster: @media_attachment.thumbnail.present? ? @media_attachment.thumbnail.url : @media_attachment.account.avatar_static_url, backgroundColor: @media_attachment.file.meta.dig('colors', 'background'), foregroundColor: @media_attachment.file.meta.dig('colors', 'foreground'), accentColor: @media_attachment.file.meta.dig('colors', 'accent'), width: 670, height: 380, fullscreen: true, alt: @media_attachment.description, duration: @media_attachment.file.meta.dig(:original, :duration) do + = react_component :audio, src: @media_attachment.file.url(:original), poster: @media_attachment.thumbnail.present? ? @media_attachment.thumbnail.url : @media_attachment.account.avatar_static_url, backgroundColor: meta.dig('colors', 'background'), foregroundColor: meta.dig('colors', 'foreground'), accentColor: meta.dig('colors', 'accent'), width: 670, height: 380, fullscreen: true, alt: @media_attachment.description, duration: meta.dig(:original, :duration) do %audio{ controls: 'controls' } %source{ src: @media_attachment.file.url(:original) } diff --git a/app/views/statuses/_detailed_status.html.haml b/app/views/statuses/_detailed_status.html.haml index 93af131e5..daf164949 100644 --- a/app/views/statuses/_detailed_status.html.haml +++ b/app/views/statuses/_detailed_status.html.haml @@ -2,13 +2,13 @@ .p-author.h-card = link_to ActivityPub::TagManager.instance.url_for(status.account), class: 'detailed-status__display-name u-url', target: stream_link_target, rel: 'noopener' do .detailed-status__display-avatar - - if current_account&.user&.setting_auto_play_gif || autoplay + - if prefers_autoplay? = image_tag status.account.avatar_original_url, alt: '', class: 'account__avatar u-photo' - else = image_tag status.account.avatar_static_url, alt: '', class: 'account__avatar u-photo' %span.display-name %bdi - %strong.display-name__html.p-name.emojify= display_name(status.account, custom_emojify: true, autoplay: autoplay) + %strong.display-name__html.p-name.emojify= display_name(status.account, custom_emojify: true, autoplay: prefers_autoplay?) %span.display-name__account = acct(status.account) = fa_icon('lock') if status.account.locked? @@ -18,28 +18,22 @@ .status__content.emojify{ :data => ({ spoiler: current_account&.user&.setting_expand_spoilers ? 'expanded' : 'folded' } if status.spoiler_text?) }< - if status.spoiler_text? %p< - %span.p-summary> #{Formatter.instance.format_spoiler(status, autoplay: autoplay)}  + %span.p-summary> #{Formatter.instance.format_spoiler(status, autoplay: prefers_autoplay?)}  %button.status__content__spoiler-link= t('statuses.show_more') .e-content - = Formatter.instance.format(status, custom_emojify: true, autoplay: autoplay) + = Formatter.instance.format(status, custom_emojify: true, autoplay: prefers_autoplay?) - if status.preloadable_poll - = react_component :poll, disabled: true, poll: ActiveModelSerializers::SerializableResource.new(status.preloadable_poll, serializer: REST::PollSerializer, scope: current_user, scope_name: :current_user).as_json do - = render partial: 'statuses/poll', locals: { status: status, poll: status.preloadable_poll, autoplay: autoplay } + = render_poll_component(status) - if !status.media_attachments.empty? - if status.media_attachments.first.video? - - video = status.media_attachments.first - = react_component :video, src: full_asset_url(video.file.url(:original)), preview: full_asset_url(video.thumbnail.present? ? video.thumbnail.url : video.file.url(:small)), frameRate: video.file.meta.dig('original', 'frame_rate'), blurhash: video.blurhash, sensitive: sensitized?(status, current_account), width: 670, height: 380, detailed: true, inline: true, alt: video.description, media: [ActiveModelSerializers::SerializableResource.new(video, serializer: REST::MediaAttachmentSerializer)].as_json do - = render partial: 'statuses/attachment_list', locals: { attachments: status.media_attachments } + = render_video_component(status, width: 670, height: 380, detailed: true) - elsif status.media_attachments.first.audio? - - audio = status.media_attachments.first - = react_component :audio, src: full_asset_url(audio.file.url(:original)), poster: full_asset_url(audio.thumbnail.present? ? audio.thumbnail.url : status.account.avatar_static_url), backgroundColor: audio.file.meta.dig('colors', 'background'), foregroundColor: audio.file.meta.dig('colors', 'foreground'), accentColor: audio.file.meta.dig('colors', 'accent'), width: 670, height: 380, alt: audio.description, duration: audio.file.meta.dig('original', 'duration') do - = render partial: 'statuses/attachment_list', locals: { attachments: status.media_attachments } + = render_audio_component(status, width: 670, height: 380) - else - = react_component :media_gallery, height: 380, sensitive: sensitized?(status, current_account), standalone: true, autoplay: autoplay, media: status.media_attachments.map { |a| ActiveModelSerializers::SerializableResource.new(a, serializer: REST::MediaAttachmentSerializer).as_json } do - = render partial: 'statuses/attachment_list', locals: { attachments: status.media_attachments } + = render_media_gallery_component(status, height: 380, standalone: true) - elsif status.preview_card - = react_component :card, sensitive: sensitized?(status, current_account), 'maxDescription': 160, card: ActiveModelSerializers::SerializableResource.new(status.preview_card, serializer: REST::PreviewCardSerializer).as_json + = render_card_component(status) .detailed-status__meta %data.dt-published{ value: status.created_at.to_time.iso8601 } diff --git a/app/views/statuses/_poll.html.haml b/app/views/statuses/_poll.html.haml index 64e62e97c..3546a923e 100644 --- a/app/views/statuses/_poll.html.haml +++ b/app/views/statuses/_poll.html.haml @@ -12,7 +12,7 @@ %span.poll__number>< = "#{percent.round}%" %span.poll__option__text - = Formatter.instance.format_poll_option(status, option, autoplay: autoplay) + = Formatter.instance.format_poll_option(status, option, autoplay: prefers_autoplay?) - if own_votes.include?(index) %span.poll__voted %i.poll__voted__mark.fa.fa-check @@ -23,7 +23,7 @@ %label.poll__option>< %span.poll__input{ class: poll.multiple? ? 'checkbox' : nil}>< %span.poll__option__text - = Formatter.instance.format_poll_option(status, option, autoplay: autoplay) + = Formatter.instance.format_poll_option(status, option, autoplay: prefers_autoplay?) .poll__footer - unless show_results %button.button.button-secondary{ disabled: true } diff --git a/app/views/statuses/_simple_status.html.haml b/app/views/statuses/_simple_status.html.haml index 7e5f11259..728e6b9b0 100644 --- a/app/views/statuses/_simple_status.html.haml +++ b/app/views/statuses/_simple_status.html.haml @@ -13,13 +13,13 @@ = link_to ActivityPub::TagManager.instance.url_for(status.account), class: 'status__display-name u-url', target: stream_link_target, rel: 'noopener noreferrer' do .status__avatar %div - - if current_account&.user&.setting_auto_play_gif || autoplay + - if prefers_autoplay? = image_tag status.account.avatar_original_url, alt: '', class: 'u-photo account__avatar' - else = image_tag status.account.avatar_static_url, alt: '', class: 'u-photo account__avatar' %span.display-name %bdi - %strong.display-name__html.p-name.emojify= display_name(status.account, custom_emojify: true, autoplay: autoplay) + %strong.display-name__html.p-name.emojify= display_name(status.account, custom_emojify: true, autoplay: prefers_autoplay?) = ' ' %span.display-name__account = acct(status.account) @@ -27,28 +27,22 @@ .status__content.emojify{ :data => ({ spoiler: current_account&.user&.setting_expand_spoilers ? 'expanded' : 'folded' } if status.spoiler_text?) }< - if status.spoiler_text? %p< - %span.p-summary> #{Formatter.instance.format_spoiler(status, autoplay: autoplay)}  + %span.p-summary> #{Formatter.instance.format_spoiler(status, autoplay: prefers_autoplay?)}  %button.status__content__spoiler-link= t('statuses.show_more') .e-content - = Formatter.instance.format(status, custom_emojify: true, autoplay: autoplay) + = Formatter.instance.format(status, custom_emojify: true, autoplay: prefers_autoplay?) - if status.preloadable_poll - = react_component :poll, disabled: true, poll: ActiveModelSerializers::SerializableResource.new(status.preloadable_poll, serializer: REST::PollSerializer, scope: current_user, scope_name: :current_user).as_json do - = render partial: 'statuses/poll', locals: { status: status, poll: status.preloadable_poll, autoplay: autoplay } + = render_poll_component(status) - if !status.media_attachments.empty? - if status.media_attachments.first.video? - - video = status.media_attachments.first - = react_component :video, src: full_asset_url(video.file.url(:original)), preview: full_asset_url(video.thumbnail.present? ? video.thumbnail.url : video.file.url(:small)), frameRate: video.file.meta.dig('original', 'frame_rate'), blurhash: video.blurhash, sensitive: sensitized?(status, current_account), width: 610, height: 343, inline: true, alt: video.description, media: [ActiveModelSerializers::SerializableResource.new(video, serializer: REST::MediaAttachmentSerializer)].as_json do - = render partial: 'statuses/attachment_list', locals: { attachments: status.media_attachments } + = render_video_component(status, width: 610, height: 343) - elsif status.media_attachments.first.audio? - - audio = status.media_attachments.first - = react_component :audio, src: full_asset_url(audio.file.url(:original)), poster: full_asset_url(audio.thumbnail.present? ? audio.thumbnail.url : status.account.avatar_static_url), backgroundColor: audio.file.meta.dig('colors', 'background'), foregroundColor: audio.file.meta.dig('colors', 'foreground'), accentColor: audio.file.meta.dig('colors', 'accent'), width: 610, height: 343, alt: audio.description, duration: audio.file.meta.dig('original', 'duration') do - = render partial: 'statuses/attachment_list', locals: { attachments: status.media_attachments } + = render_audio_component(status, width: 610, height: 343) - else - = react_component :media_gallery, height: 343, sensitive: sensitized?(status, current_account), autoplay: autoplay, media: status.media_attachments.map { |a| ActiveModelSerializers::SerializableResource.new(a, serializer: REST::MediaAttachmentSerializer).as_json } do - = render partial: 'statuses/attachment_list', locals: { attachments: status.media_attachments } + = render_media_gallery_component(status, height: 343) - elsif status.preview_card - = react_component :card, sensitive: sensitized?(status, current_account), 'maxDescription': 160, card: ActiveModelSerializers::SerializableResource.new(status.preview_card, serializer: REST::PreviewCardSerializer).as_json + = render_card_component(status) - if !status.in_reply_to_id.nil? && status.in_reply_to_account_id == status.account.id && !hide_show_thread = link_to ActivityPub::TagManager.instance.url_for(status), class: 'status__content__read-more-button', target: stream_link_target, rel: 'noopener noreferrer' do diff --git a/app/views/statuses/_status.html.haml b/app/views/statuses/_status.html.haml index 13a06519c..9f3197d0d 100644 --- a/app/views/statuses/_status.html.haml +++ b/app/views/statuses/_status.html.haml @@ -5,7 +5,6 @@ is_successor ||= false direct_reply_id ||= false parent_id ||= false - autoplay ||= current_account&.user&.setting_auto_play_gif is_direct_parent = direct_reply_id == status.id is_direct_child = parent_id == status.in_reply_to_id centered ||= include_threads && !is_predecessor && !is_successor @@ -19,7 +18,7 @@ .entry{ class: entry_classes } = link_to_older ActivityPub::TagManager.instance.url_for(@next_ancestor) - = render partial: 'statuses/status', collection: @ancestors, as: :status, locals: { is_predecessor: true, direct_reply_id: status.in_reply_to_id }, autoplay: autoplay + = render partial: 'statuses/status', collection: @ancestors, as: :status, locals: { is_predecessor: true, direct_reply_id: status.in_reply_to_id } .entry{ class: entry_classes } @@ -39,14 +38,14 @@ %span = t('stream_entries.pinned') - = render (centered ? 'statuses/detailed_status' : 'statuses/simple_status'), status: status.proper, autoplay: autoplay, hide_show_thread: is_predecessor || is_successor + = render (centered ? 'statuses/detailed_status' : 'statuses/simple_status'), status: status.proper, hide_show_thread: is_predecessor || is_successor - if include_threads - if @since_descendant_thread_id .entry{ class: entry_classes } = link_to_newer short_account_status_url(status.account.username, status, max_descendant_thread_id: @since_descendant_thread_id + 1) - @descendant_threads.each do |thread| - = render partial: 'statuses/status', collection: thread[:statuses], as: :status, locals: { is_successor: true, parent_id: status.id }, autoplay: autoplay + = render partial: 'statuses/status', collection: thread[:statuses], as: :status, locals: { is_successor: true, parent_id: status.id } - if thread[:next_status] .entry{ class: entry_classes } diff --git a/app/views/statuses/embed.html.haml b/app/views/statuses/embed.html.haml index 2f111f53f..18d62fd8e 100644 --- a/app/views/statuses/embed.html.haml +++ b/app/views/statuses/embed.html.haml @@ -1,2 +1,2 @@ .activity-stream.activity-stream--headless - = render 'status', status: @status, centered: true, autoplay: @autoplay + = render 'status', status: @status, centered: true -- cgit From 7cb34b32f8bc925b56c79dbcf053671f93f2eb42 Mon Sep 17 00:00:00 2001 From: Takeshi Umeda Date: Thu, 6 May 2021 06:39:02 +0900 Subject: Add management of delivery availability in Federation settings (#15771) * Add management of delivery availavility in Federation settings * fix translate * Remove useless object creation * Fix DeepSource issue * Add shortcut for all * Fix DeepSource(skipcq) * Change 'remove' to 'clear' * Fix style * Change class method name (exhausted_deliveries_key_by) --- app/controllers/admin/instances_controller.rb | 44 +++++++++++++++++++++- app/helpers/admin/action_logs_helper.rb | 4 +- app/lib/delivery_failure_tracker.rb | 26 +++++++++++++ app/models/admin/action_log_filter.rb | 2 + app/models/instance.rb | 3 ++ app/models/instance_filter.rb | 8 +++- app/policies/delivery_policy.rb | 15 ++++++++ .../instances/_exhausted_deliveries_days.haml | 2 + app/views/admin/instances/_instance.html.haml | 8 ++++ app/views/admin/instances/index.html.haml | 18 +++++++++ app/views/admin/instances/show.html.haml | 25 ++++++++++++ config/locales/en.yml | 21 +++++++++++ config/routes.rb | 9 ++++- 13 files changed, 180 insertions(+), 5 deletions(-) create mode 100644 app/policies/delivery_policy.rb create mode 100644 app/views/admin/instances/_exhausted_deliveries_days.haml (limited to 'app/controllers') diff --git a/app/controllers/admin/instances_controller.rb b/app/controllers/admin/instances_controller.rb index b5918d231..748c5de5a 100644 --- a/app/controllers/admin/instances_controller.rb +++ b/app/controllers/admin/instances_controller.rb @@ -3,7 +3,8 @@ module Admin class InstancesController < BaseController before_action :set_instances, only: :index - before_action :set_instance, only: :show + before_action :set_instance, except: :index + before_action :set_exhausted_deliveries_days, only: :show def index authorize :instance, :index? @@ -13,14 +14,55 @@ module Admin authorize :instance, :show? end + def clear_delivery_errors + authorize :delivery, :clear_delivery_errors? + + @instance.delivery_failure_tracker.clear_failures! + redirect_to admin_instance_path(@instance.domain) + end + + def restart_delivery + authorize :delivery, :restart_delivery? + + last_unavailable_domain = unavailable_domain + + if last_unavailable_domain.present? + @instance.delivery_failure_tracker.track_success! + log_action :destroy, last_unavailable_domain + end + + redirect_to admin_instance_path(@instance.domain) + end + + def stop_delivery + authorize :delivery, :stop_delivery? + + UnavailableDomain.create(domain: @instance.domain) + log_action :create, unavailable_domain + redirect_to admin_instance_path(@instance.domain) + end + private def set_instance @instance = Instance.find(params[:id]) end + def set_exhausted_deliveries_days + @exhausted_deliveries_days = @instance.delivery_failure_tracker.exhausted_deliveries_days + end + def set_instances @instances = filtered_instances.page(params[:page]) + warning_domains_map = DeliveryFailureTracker.warning_domains_map + + @instances.each do |instance| + instance.failure_days = warning_domains_map[instance.domain] + end + end + + def unavailable_domain + UnavailableDomain.find_by(domain: @instance.domain) end def filtered_instances diff --git a/app/helpers/admin/action_logs_helper.rb b/app/helpers/admin/action_logs_helper.rb index 0f3ca36e2..e9a298a24 100644 --- a/app/helpers/admin/action_logs_helper.rb +++ b/app/helpers/admin/action_logs_helper.rb @@ -21,7 +21,7 @@ module Admin::ActionLogsHelper record.shortcode when 'Report' link_to "##{record.id}", admin_report_path(record) - when 'DomainBlock', 'DomainAllow', 'EmailDomainBlock' + when 'DomainBlock', 'DomainAllow', 'EmailDomainBlock', 'UnavailableDomain' link_to record.domain, "https://#{record.domain}" when 'Status' link_to record.account.acct, ActivityPub::TagManager.instance.url_for(record) @@ -38,7 +38,7 @@ module Admin::ActionLogsHelper case type when 'CustomEmoji' attributes['shortcode'] - when 'DomainBlock', 'DomainAllow', 'EmailDomainBlock' + when 'DomainBlock', 'DomainAllow', 'EmailDomainBlock', 'UnavailableDomain' link_to attributes['domain'], "https://#{attributes['domain']}" when 'Status' tmp_status = Status.new(attributes.except('reblogs_count', 'favourites_count')) diff --git a/app/lib/delivery_failure_tracker.rb b/app/lib/delivery_failure_tracker.rb index 2cd6ef7ad..8907ade4c 100644 --- a/app/lib/delivery_failure_tracker.rb +++ b/app/lib/delivery_failure_tracker.rb @@ -17,6 +17,10 @@ class DeliveryFailureTracker UnavailableDomain.find_by(domain: @host)&.destroy end + def clear_failures! + Redis.current.del(exhausted_deliveries_key) + end + def days Redis.current.scard(exhausted_deliveries_key) || 0 end @@ -25,6 +29,10 @@ class DeliveryFailureTracker !UnavailableDomain.where(domain: @host).exists? end + def exhausted_deliveries_days + Redis.current.smembers(exhausted_deliveries_key).sort.map { |date| Date.new(date.slice(0, 4).to_i, date.slice(4, 2).to_i, date.slice(6, 2).to_i) } + end + alias reset! track_success! class << self @@ -44,6 +52,24 @@ class DeliveryFailureTracker def reset!(url) new(url).reset! end + + def warning_domains + domains = Redis.current.keys(exhausted_deliveries_key_by('*')).map do |key| + key.delete_prefix(exhausted_deliveries_key_by('')) + end + + domains - UnavailableDomain.all.pluck(:domain) + end + + def warning_domains_map + warning_domains.index_with { |domain| Redis.current.scard(exhausted_deliveries_key_by(domain)) } + end + + private + + def exhausted_deliveries_key_by(host) + "exhausted_deliveries:#{host}" + end end private diff --git a/app/models/admin/action_log_filter.rb b/app/models/admin/action_log_filter.rb index 3a1b67e06..a1c156a8b 100644 --- a/app/models/admin/action_log_filter.rb +++ b/app/models/admin/action_log_filter.rb @@ -17,12 +17,14 @@ class Admin::ActionLogFilter create_domain_allow: { target_type: 'DomainAllow', action: 'create' }.freeze, create_domain_block: { target_type: 'DomainBlock', action: 'create' }.freeze, create_email_domain_block: { target_type: 'EmailDomainBlock', action: 'create' }.freeze, + create_unavailable_domain: { target_type: 'UnavailableDomain', action: 'create' }.freeze, demote_user: { target_type: 'User', action: 'demote' }.freeze, destroy_announcement: { target_type: 'Announcement', action: 'destroy' }.freeze, destroy_custom_emoji: { target_type: 'CustomEmoji', action: 'destroy' }.freeze, destroy_domain_allow: { target_type: 'DomainAllow', action: 'destroy' }.freeze, destroy_domain_block: { target_type: 'DomainBlock', action: 'destroy' }.freeze, destroy_email_domain_block: { target_type: 'EmailDomainBlock', action: 'destroy' }.freeze, + destroy_unavailable_domain: { target_type: 'UnavailableDomain', action: 'destroy' }.freeze, destroy_status: { target_type: 'Status', action: 'destroy' }.freeze, disable_2fa_user: { target_type: 'User', action: 'disable' }.freeze, disable_custom_emoji: { target_type: 'CustomEmoji', action: 'disable' }.freeze, diff --git a/app/models/instance.rb b/app/models/instance.rb index 29be03662..8949be054 100644 --- a/app/models/instance.rb +++ b/app/models/instance.rb @@ -10,10 +10,13 @@ class Instance < ApplicationRecord self.primary_key = :domain + attr_accessor :failure_days + has_many :accounts, foreign_key: :domain, primary_key: :domain belongs_to :domain_block, foreign_key: :domain, primary_key: :domain belongs_to :domain_allow, foreign_key: :domain, primary_key: :domain + belongs_to :unavailable_domain, foreign_key: :domain, primary_key: :domain # skipcq: RB-RL1031 scope :matches_domain, ->(value) { where(arel_table[:domain].matches("%#{value}%")) } diff --git a/app/models/instance_filter.rb b/app/models/instance_filter.rb index 0598d8fea..9e533c4aa 100644 --- a/app/models/instance_filter.rb +++ b/app/models/instance_filter.rb @@ -4,6 +4,8 @@ class InstanceFilter KEYS = %i( limited by_domain + warning + unavailable ).freeze attr_reader :params @@ -13,7 +15,7 @@ class InstanceFilter end def results - scope = Instance.includes(:domain_block, :domain_allow).order(accounts_count: :desc) + scope = Instance.includes(:domain_block, :domain_allow, :unavailable_domain).order(accounts_count: :desc) params.each do |key, value| scope.merge!(scope_for(key, value.to_s.strip)) if value.present? @@ -32,6 +34,10 @@ class InstanceFilter Instance.joins(:domain_allow).reorder(Arel.sql('domain_allows.id desc')) when 'by_domain' Instance.matches_domain(value) + when 'warning' + Instance.where(domain: DeliveryFailureTracker.warning_domains) + when 'unavailable' + Instance.joins(:unavailable_domain) else raise "Unknown filter: #{key}" end diff --git a/app/policies/delivery_policy.rb b/app/policies/delivery_policy.rb new file mode 100644 index 000000000..24d06c168 --- /dev/null +++ b/app/policies/delivery_policy.rb @@ -0,0 +1,15 @@ +# frozen_string_literal: true + +class DeliveryPolicy < ApplicationPolicy + def clear_delivery_errors? + admin? + end + + def restart_delivery? + admin? + end + + def stop_delivery? + admin? + end +end diff --git a/app/views/admin/instances/_exhausted_deliveries_days.haml b/app/views/admin/instances/_exhausted_deliveries_days.haml new file mode 100644 index 000000000..e581f542d --- /dev/null +++ b/app/views/admin/instances/_exhausted_deliveries_days.haml @@ -0,0 +1,2 @@ +%li.negative-hint + = l(exhausted_deliveries_days) diff --git a/app/views/admin/instances/_instance.html.haml b/app/views/admin/instances/_instance.html.haml index 188d0d984..990cf9ec8 100644 --- a/app/views/admin/instances/_instance.html.haml +++ b/app/views/admin/instances/_instance.html.haml @@ -22,4 +22,12 @@ = t('admin.accounts.whitelisted') - else = t('admin.accounts.no_limits_imposed') + - if instance.failure_days + = ' / ' + %span.negative-hint + = t('admin.instances.delivery.warning_message', count: instance.failure_days) + - if instance.unavailable_domain + = ' / ' + %span.negative-hint + = t('admin.instances.delivery.unavailable_message') .trends__item__current{ title: t('admin.instances.known_accounts', count: instance.accounts_count) }= number_to_human instance.accounts_count, strip_insignificant_zeros: true diff --git a/app/views/admin/instances/index.html.haml b/app/views/admin/instances/index.html.haml index 7c7958786..797948d94 100644 --- a/app/views/admin/instances/index.html.haml +++ b/app/views/admin/instances/index.html.haml @@ -16,6 +16,24 @@ - unless whitelist_mode? %li= filter_link_to t('admin.instances.moderation.limited'), limited: '1' + .filter-subset + %strong= t('admin.instances.delivery.title') + %ul + %li= filter_link_to t('admin.instances.delivery.all'), warning: nil, unavailable: nil + %li= filter_link_to t('admin.instances.delivery.warning'), warning: '1', unavailable: nil + %li= filter_link_to t('admin.instances.delivery.unavailable'), warning: nil, unavailable: '1' + + .back-link + = link_to admin_instances_path() do + %i.fa.fa-chevron-left.fa-fw + = t('admin.instances.back_to_all') + = link_to admin_instances_path(limited: 1) do + %i.fa.fa-chevron-left.fa-fw + = t('admin.instances.back_to_limited') + = link_to admin_instances_path(warning: 1) do + %i.fa.fa-chevron-left.fa-fw + = t('admin.instances.back_to_warning') + - unless whitelist_mode? = form_tag admin_instances_url, method: 'GET', class: 'simple_form' do .fields-group diff --git a/app/views/admin/instances/show.html.haml b/app/views/admin/instances/show.html.haml index 0b9382771..462529338 100644 --- a/app/views/admin/instances/show.html.haml +++ b/app/views/admin/instances/show.html.haml @@ -1,6 +1,18 @@ - content_for :page_title do = @instance.domain +.filters + .back-link + = link_to admin_instances_path() do + %i.fa.fa-chevron-left.fa-fw + = t('admin.instances.back_to_all') + = link_to admin_instances_path(limited: 1) do + %i.fa.fa-chevron-left.fa-fw + = t('admin.instances.back_to_limited') + = link_to admin_instances_path(warning: 1) do + %i.fa.fa-chevron-left.fa-fw + = t('admin.instances.back_to_warning') + .dashboard__counters %div = link_to admin_accounts_path(remote: '1', by_domain: @instance.domain) do @@ -48,6 +60,13 @@ = simple_format(h(@instance.public_comment)) .speech-bubble__owner= t 'admin.instances.public_comment' +- unless @exhausted_deliveries_days.empty? + %h4= t 'admin.instances.delivery_error_days' + %ul + = render partial: 'exhausted_deliveries_days', collection: @exhausted_deliveries_days + %p.hint + = t 'admin.instances.delivery_error_hint', count: DeliveryFailureTracker::FAILURE_DAYS_THRESHOLD + %hr.spacer/ %div.action-buttons @@ -59,3 +78,9 @@ = link_to t('admin.domain_blocks.undo'), admin_domain_block_path(@instance.domain_block), class: 'button' - else = link_to t('admin.domain_blocks.add_new'), new_admin_domain_block_path(_domain: @instance.domain), class: 'button' + - if @instance.delivery_failure_tracker.available? + - unless @exhausted_deliveries_days.empty? + = link_to t('admin.instances.delivery.clear'), clear_delivery_errors_admin_instance_path(@instance), data: { confirm: t('admin.accounts.are_you_sure'), method: :post }, class: 'button' + = link_to t('admin.instances.delivery.stop'), stop_delivery_admin_instance_path(@instance), data: { confirm: t('admin.accounts.are_you_sure'), method: :post }, class: 'button' + - else + = link_to t('admin.instances.delivery.restart'), restart_delivery_admin_instance_path(@instance), data: { confirm: t('admin.accounts.are_you_sure'), method: :post }, class: 'button' diff --git a/config/locales/en.yml b/config/locales/en.yml index 1b41ee063..bfa489817 100644 --- a/config/locales/en.yml +++ b/config/locales/en.yml @@ -230,6 +230,7 @@ en: create_domain_block: Create Domain Block create_email_domain_block: Create E-mail Domain Block create_ip_block: Create IP rule + create_unavailable_domain: Create Unavailable Domain demote_user: Demote User destroy_announcement: Delete Announcement destroy_custom_emoji: Delete Custom Emoji @@ -238,6 +239,7 @@ en: destroy_email_domain_block: Delete e-mail domain block destroy_ip_block: Delete IP rule destroy_status: Delete Post + destroy_unavailable_domain: Delete Unavailable Domain disable_2fa_user: Disable 2FA disable_custom_emoji: Disable Custom Emoji disable_user: Disable User @@ -271,6 +273,7 @@ en: create_domain_block_html: "%{name} blocked domain %{target}" create_email_domain_block_html: "%{name} blocked e-mail domain %{target}" create_ip_block_html: "%{name} created rule for IP %{target}" + create_unavailable_domain_html: "%{name} stopped delivery to domain %{target}" demote_user_html: "%{name} demoted user %{target}" destroy_announcement_html: "%{name} deleted announcement %{target}" destroy_custom_emoji_html: "%{name} destroyed emoji %{target}" @@ -279,6 +282,7 @@ en: destroy_email_domain_block_html: "%{name} unblocked e-mail domain %{target}" destroy_ip_block_html: "%{name} deleted rule for IP %{target}" destroy_status_html: "%{name} removed post by %{target}" + destroy_unavailable_domain_html: "%{name} resumed delivery to domain %{target}" disable_2fa_user_html: "%{name} disabled two factor requirement for user %{target}" disable_custom_emoji_html: "%{name} disabled emoji %{target}" disable_user_html: "%{name} disabled login for user %{target}" @@ -451,8 +455,25 @@ en: title: Follow recommendations unsuppress: Restore follow recommendation instances: + back_to_all: All + back_to_limited: Limited + back_to_warning: Warning by_domain: Domain + delivery: + all: All + clear: Clear delivery errors + restart: Restart delivery + stop: Stop delivery + title: Delivery + unavailable: Unavailable + unavailable_message: Delivery unavailable + warning: Warning + warning_message: + one: Delivery failure %{count} day + other: Delivery failure %{count} days delivery_available: Delivery is available + delivery_error_days: Delivery error days + delivery_error_hint: If delivery is not possible for %{count} days, it will be automatically marked as undeliverable. empty: No domains found. known_accounts: one: "%{count} known account" diff --git a/config/routes.rb b/config/routes.rb index 4661a7c11..8ca7fccdd 100644 --- a/config/routes.rb +++ b/config/routes.rb @@ -217,7 +217,14 @@ Rails.application.routes.draw do end end - resources :instances, only: [:index, :show], constraints: { id: /[^\/]+/ } + resources :instances, only: [:index, :show], constraints: { id: /[^\/]+/ } do + member do + post :clear_delivery_errors + post :restart_delivery + post :stop_delivery + end + end + resources :rules resources :reports, only: [:index, :show] do -- cgit From 566fc909134586d1746ad60ee455832dec6bc61a Mon Sep 17 00:00:00 2001 From: Claire Date: Thu, 6 May 2021 14:22:54 +0200 Subject: Add Ruby 3.0 support (#16046) * Fix issues with POSIX::Spawn, Terrapin and Ruby 3.0 Also improve the Terrapin monkey-patch for the stderr/stdout issue. * Fix keyword argument handling throughout the codebase * Monkey-patch Paperclip to fix keyword arguments handling in validators * Change validation_extensions to please CodeClimate * Bump microformats from 4.2.1 to 4.3.1 * Allow Ruby 3.0 * Add Ruby 3.0 test target to CircleCI * Add test for admin dashboard warnings * Fix admin dashboard warnings on Ruby 3.0 --- .circleci/config.yml | 27 +++++++ Gemfile | 2 +- Gemfile.lock | 6 +- app/controllers/activitypub/outboxes_controller.rb | 2 +- app/controllers/api/v1/accounts_controller.rb | 4 +- .../api/v1/follow_requests_controller.rb | 2 +- app/models/session_activation.rb | 2 +- app/models/user.rb | 19 +++-- app/views/admin/dashboard/index.html.haml | 2 +- app/workers/import/relationship_worker.rb | 6 +- config/application.rb | 1 + config/initializers/session_store.rb | 5 +- lib/paperclip/validation_extensions.rb | 58 +++++++++++++++ lib/terrapin/multi_pipe_extensions.rb | 87 +++++++++++----------- .../controllers/admin/dashboard_controller_spec.rb | 12 ++- spec/mailers/notification_mailer_spec.rb | 4 +- spec/mailers/user_mailer_spec.rb | 4 +- spec/models/session_activation_spec.rb | 6 +- .../account_relationships_presenter_spec.rb | 2 +- 19 files changed, 177 insertions(+), 74 deletions(-) create mode 100644 lib/paperclip/validation_extensions.rb (limited to 'app/controllers') diff --git a/.circleci/config.yml b/.circleci/config.yml index 862fa126b..2f3860d7c 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -129,6 +129,13 @@ jobs: environment: *ruby_environment <<: *install_ruby_dependencies + install-ruby3.0: + <<: *defaults + docker: + - image: circleci/ruby:3.0-buster-node + environment: *ruby_environment + <<: *install_ruby_dependencies + build: <<: *defaults steps: @@ -187,6 +194,18 @@ jobs: - image: circleci/redis:5-alpine <<: *test_steps + test-ruby3.0: + <<: *defaults + docker: + - image: circleci/ruby:3.0-buster-node + environment: *ruby_environment + - image: circleci/postgres:12.2 + environment: + POSTGRES_USER: root + POSTGRES_HOST_AUTH_METHOD: trust + - image: circleci/redis:5-alpine + <<: *test_steps + test-webui: <<: *defaults docker: @@ -227,6 +246,10 @@ workflows: requires: - install - install-ruby2.7 + - install-ruby3.0: + requires: + - install + - install-ruby2.7 - build: requires: - install-ruby2.7 @@ -241,6 +264,10 @@ workflows: requires: - install-ruby2.6 - build + - test-ruby3.0: + requires: + - install-ruby3.0 + - build - test-webui: requires: - install diff --git a/Gemfile b/Gemfile index 6ca0a81de..5a55d6b04 100644 --- a/Gemfile +++ b/Gemfile @@ -1,7 +1,7 @@ # frozen_string_literal: true source 'https://rubygems.org' -ruby '>= 2.5.0', '< 3.0.0' +ruby '>= 2.5.0', '< 3.1.0' gem 'pkg-config', '~> 1.4' diff --git a/Gemfile.lock b/Gemfile.lock index b1ae4fd22..980750b63 100644 --- a/Gemfile.lock +++ b/Gemfile.lock @@ -292,7 +292,7 @@ GEM ipaddress (0.8.3) iso-639 (0.3.5) jmespath (1.4.0) - json (2.3.1) + json (2.5.1) json-canonicalization (0.2.1) json-ld (3.1.9) htmlentities (~> 4.3) @@ -344,7 +344,7 @@ GEM redis (>= 3.0.5) memory_profiler (1.0.0) method_source (1.0.0) - microformats (4.2.1) + microformats (4.3.1) json (~> 2.2) nokogiri (~> 1.10) mime-types (3.3.1) @@ -354,7 +354,7 @@ GEM nokogiri (~> 1) rake mini_mime (1.0.3) - mini_portile2 (2.5.0) + mini_portile2 (2.5.1) minitest (5.14.4) msgpack (1.4.2) multi_json (1.15.0) diff --git a/app/controllers/activitypub/outboxes_controller.rb b/app/controllers/activitypub/outboxes_controller.rb index 5fd735ad6..111285036 100644 --- a/app/controllers/activitypub/outboxes_controller.rb +++ b/app/controllers/activitypub/outboxes_controller.rb @@ -20,7 +20,7 @@ class ActivityPub::OutboxesController < ActivityPub::BaseController def outbox_presenter if page_requested? ActivityPub::CollectionPresenter.new( - id: outbox_url(page_params), + id: outbox_url(**page_params), type: :ordered, part_of: outbox_url, prev: prev_page, diff --git a/app/controllers/api/v1/accounts_controller.rb b/app/controllers/api/v1/accounts_controller.rb index 996f1b79b..95869f554 100644 --- a/app/controllers/api/v1/accounts_controller.rb +++ b/app/controllers/api/v1/accounts_controller.rb @@ -35,7 +35,7 @@ class Api::V1::AccountsController < Api::BaseController follow = FollowService.new.call(current_user.account, @account, reblogs: params.key?(:reblogs) ? truthy_param?(:reblogs) : nil, notify: params.key?(:notify) ? truthy_param?(:notify) : nil, with_rate_limit: true) options = @account.locked? || current_user.account.silenced? ? {} : { following_map: { @account.id => { reblogs: follow.show_reblogs?, notify: follow.notify? } }, requested_map: { @account.id => false } } - render json: @account, serializer: REST::RelationshipSerializer, relationships: relationships(options) + render json: @account, serializer: REST::RelationshipSerializer, relationships: relationships(**options) end def block @@ -70,7 +70,7 @@ class Api::V1::AccountsController < Api::BaseController end def relationships(**options) - AccountRelationshipsPresenter.new([@account.id], current_user.account_id, options) + AccountRelationshipsPresenter.new([@account.id], current_user.account_id, **options) end def account_params diff --git a/app/controllers/api/v1/follow_requests_controller.rb b/app/controllers/api/v1/follow_requests_controller.rb index b34c76f29..f4b2a74d0 100644 --- a/app/controllers/api/v1/follow_requests_controller.rb +++ b/app/controllers/api/v1/follow_requests_controller.rb @@ -29,7 +29,7 @@ class Api::V1::FollowRequestsController < Api::BaseController end def relationships(**options) - AccountRelationshipsPresenter.new([params[:id]], current_user.account_id, options) + AccountRelationshipsPresenter.new([params[:id]], current_user.account_id, **options) end def load_accounts diff --git a/app/models/session_activation.rb b/app/models/session_activation.rb index b0ce9d112..3a59bad93 100644 --- a/app/models/session_activation.rb +++ b/app/models/session_activation.rb @@ -44,7 +44,7 @@ class SessionActivation < ApplicationRecord end def activate(**options) - activation = create!(options) + activation = create!(**options) purge_old activation end diff --git a/app/models/user.rb b/app/models/user.rb index 0440627c5..4973c68b6 100644 --- a/app/models/user.rb +++ b/app/models/user.rb @@ -370,15 +370,20 @@ class User < ApplicationRecord protected - def send_devise_notification(notification, *args) + def send_devise_notification(notification, *args, **kwargs) # This method can be called in `after_update` and `after_commit` hooks, # but we must make sure the mailer is actually called *after* commit, # otherwise it may work on stale data. To do this, figure out if we are # within a transaction. + + # It seems like devise sends keyword arguments as a hash in the last + # positional argument + kwargs = args.pop if args.last.is_a?(Hash) && kwargs.empty? + if ActiveRecord::Base.connection.current_transaction.try(:records)&.include?(self) - pending_devise_notifications << [notification, args] + pending_devise_notifications << [notification, args, kwargs] else - render_and_send_devise_message(notification, *args) + render_and_send_devise_message(notification, *args, **kwargs) end end @@ -389,8 +394,8 @@ class User < ApplicationRecord end def send_pending_devise_notifications - pending_devise_notifications.each do |notification, args| - render_and_send_devise_message(notification, *args) + pending_devise_notifications.each do |notification, args, kwargs| + render_and_send_devise_message(notification, *args, **kwargs) end # Empty the pending notifications array because the @@ -403,8 +408,8 @@ class User < ApplicationRecord @pending_devise_notifications ||= [] end - def render_and_send_devise_message(notification, *args) - devise_mailer.send(notification, self, *args).deliver_later + def render_and_send_devise_message(notification, *args, **kwargs) + devise_mailer.send(notification, self, *args, **kwargs).deliver_later end def set_approved diff --git a/app/views/admin/dashboard/index.html.haml b/app/views/admin/dashboard/index.html.haml index 57a753e6b..e8a2b46fd 100644 --- a/app/views/admin/dashboard/index.html.haml +++ b/app/views/admin/dashboard/index.html.haml @@ -5,7 +5,7 @@ .flash-message-stack - @system_checks.each do |message| .flash-message.warning - = t("admin.system_checks.#{message.key}.message_html", message.value ? { value: content_tag(:strong, message.value) } : {}) + = t("admin.system_checks.#{message.key}.message_html", value: message.value ? content_tag(:strong, message.value) : nil) - if message.action = link_to t("admin.system_checks.#{message.key}.action"), message.action diff --git a/app/workers/import/relationship_worker.rb b/app/workers/import/relationship_worker.rb index 4a7100435..6791b15c3 100644 --- a/app/workers/import/relationship_worker.rb +++ b/app/workers/import/relationship_worker.rb @@ -5,7 +5,7 @@ class Import::RelationshipWorker sidekiq_options queue: 'pull', retry: 8, dead: false - def perform(account_id, target_account_uri, relationship, options = {}) + def perform(account_id, target_account_uri, relationship, options) from_account = Account.find(account_id) target_domain = domain(target_account_uri) target_account = stoplight_wrap_request(target_domain) { ResolveAccountService.new.call(target_account_uri, { check_delivery_availability: true }) } @@ -16,7 +16,7 @@ class Import::RelationshipWorker case relationship when 'follow' begin - FollowService.new.call(from_account, target_account, options) + FollowService.new.call(from_account, target_account, **options) rescue ActiveRecord::RecordInvalid raise if FollowLimitValidator.limit_for_account(from_account) < from_account.following_count end @@ -27,7 +27,7 @@ class Import::RelationshipWorker when 'unblock' UnblockService.new.call(from_account, target_account) when 'mute' - MuteService.new.call(from_account, target_account, options) + MuteService.new.call(from_account, target_account, **options) when 'unmute' UnmuteService.new.call(from_account, target_account) end diff --git a/config/application.rb b/config/application.rb index 37a996224..08a4e4c97 100644 --- a/config/application.rb +++ b/config/application.rb @@ -10,6 +10,7 @@ require_relative '../lib/exceptions' require_relative '../lib/enumerable' require_relative '../lib/sanitize_ext/sanitize_config' require_relative '../lib/redis/namespace_extensions' +require_relative '../lib/paperclip/validation_extensions' require_relative '../lib/paperclip/url_generator_extensions' require_relative '../lib/paperclip/attachment_extensions' require_relative '../lib/paperclip/media_type_spoof_detector_extensions' diff --git a/config/initializers/session_store.rb b/config/initializers/session_store.rb index e5d1be4c6..3d9bf96fd 100644 --- a/config/initializers/session_store.rb +++ b/config/initializers/session_store.rb @@ -1,7 +1,6 @@ # Be sure to restart your server when you modify this file. -Rails.application.config.session_store :cookie_store, { +Rails.application.config.session_store :cookie_store, key: '_mastodon_session', secure: (Rails.env.production? || ENV['LOCAL_HTTPS'] == 'true'), - same_site: :lax, -} + same_site: :lax diff --git a/lib/paperclip/validation_extensions.rb b/lib/paperclip/validation_extensions.rb new file mode 100644 index 000000000..0df0434f6 --- /dev/null +++ b/lib/paperclip/validation_extensions.rb @@ -0,0 +1,58 @@ +# frozen_string_literal: true + +# Monkey-patch various Paperclip validators for Ruby 3.0 compatibility + +module Paperclip + module Validators + module AttachmentSizeValidatorExtensions + def validate_each(record, attr_name, _value) + base_attr_name = attr_name + attr_name = "#{attr_name}_file_size".to_sym + value = record.send(:read_attribute_for_validation, attr_name) + + if value.present? + options.slice(*Paperclip::Validators::AttachmentSizeValidator::AVAILABLE_CHECKS).each do |option, option_value| + option_value = option_value.call(record) if option_value.is_a?(Proc) + option_value = extract_option_value(option, option_value) + + next if value.send(Paperclip::Validators::AttachmentSizeValidator::CHECKS[option], option_value) + + error_message_key = options[:in] ? :in_between : option + [attr_name, base_attr_name].each do |error_attr_name| + record.errors.add(error_attr_name, error_message_key, **filtered_options(value).merge( + min: min_value_in_human_size(record), + max: max_value_in_human_size(record), + count: human_size(option_value) + )) + end + end + end + end + end + + module AttachmentContentTypeValidatorExtensions + def mark_invalid(record, attribute, types) + record.errors.add attribute, :invalid, **options.merge({ types: types.join(', ') }) + end + end + + module AttachmentPresenceValidatorExtensions + def validate_each(record, attribute, _value) + if record.send("#{attribute}_file_name").blank? + record.errors.add(attribute, :blank, **options) + end + end + end + + module AttachmentFileNameValidatorExtensions + def mark_invalid(record, attribute, patterns) + record.errors.add attribute, :invalid, options.merge({ names: patterns.join(', ') }) + end + end + end +end + +Paperclip::Validators::AttachmentSizeValidator.prepend(Paperclip::Validators::AttachmentSizeValidatorExtensions) +Paperclip::Validators::AttachmentContentTypeValidator.prepend(Paperclip::Validators::AttachmentContentTypeValidatorExtensions) +Paperclip::Validators::AttachmentPresenceValidator.prepend(Paperclip::Validators::AttachmentPresenceValidatorExtensions) +Paperclip::Validators::AttachmentFileNameValidator.prepend(Paperclip::Validators::AttachmentFileNameValidatorExtensions) diff --git a/lib/terrapin/multi_pipe_extensions.rb b/lib/terrapin/multi_pipe_extensions.rb index 51d7de37c..209f4ad6c 100644 --- a/lib/terrapin/multi_pipe_extensions.rb +++ b/lib/terrapin/multi_pipe_extensions.rb @@ -1,61 +1,64 @@ # frozen_string_literal: false -# Fix adapted from https://github.com/thoughtbot/terrapin/pull/5 + +require 'fcntl' module Terrapin module MultiPipeExtensions - def read - read_streams(@stdout_in, @stderr_in) - end + def initialize + @stdout_in, @stdout_out = IO.pipe + @stderr_in, @stderr_out = IO.pipe - def close_read - begin - @stdout_in.close - rescue IOError - # Do nothing - end - - begin - @stderr_in.close - rescue IOError - # Do nothing - end + clear_nonblocking_flags! end - def read_streams(output, error) - @stdout_output = '' - @stderr_output = '' + def pipe_options + # Add some flags to explicitly close the other end of the pipes + { out: @stdout_out, err: @stderr_out, @stdout_in => :close, @stderr_in => :close } + end - read_fds = [output, error] + def read + # While we are patching Terrapin, fix child process potentially getting stuck on writing + # to stderr. - until read_fds.empty? - to_read, = IO.select(read_fds) + @stdout_output = +'' + @stderr_output = +'' - if to_read.include?(output) - @stdout_output << read_stream(output) - read_fds.delete(output) if output.closed? - end + fds_to_read = [@stdout_in, @stderr_in] + until fds_to_read.empty? + rs, = IO.select(fds_to_read) - if to_read.include?(error) - @stderr_output << read_stream(error) - read_fds.delete(error) if error.closed? - end + read_nonblocking!(@stdout_in, @stdout_output, fds_to_read) if rs.include?(@stdout_in) + read_nonblocking!(@stderr_in, @stderr_output, fds_to_read) if rs.include?(@stderr_in) end end - def read_stream(io) - result = '' - - begin - while (partial_result = io.read_nonblock(8192)) - result << partial_result - end - rescue EOFError, Errno::EPIPE - io.close - rescue Errno::EINTR, Errno::EWOULDBLOCK, Errno::EAGAIN - # Do nothing + private + + # @param [IO] io IO Stream to read until there is nothing to read + # @param [String] result Mutable string to which read values will be appended to + # @param [Array] fds_to_read Mutable array from which `io` should be removed on EOF + def read_nonblocking!(io, result, fds_to_read) + while (partial_result = io.read_nonblock(8192)) + result << partial_result end + rescue IO::WaitReadable + # Do nothing + rescue EOFError + fds_to_read.delete(io) + end + + def clear_nonblocking_flags! + # Ruby 3.0 sets pipes to non-blocking mode, and resets the flags as + # needed when calling fork/exec-related syscalls, but posix-spawn does + # not currently do that, so we need to do it manually for the time being + # so that the child process do not error out when the buffers are full. + stdout_flags = @stdout_out.fcntl(Fcntl::F_GETFL) + @stdout_out.fcntl(Fcntl::F_SETFL, stdout_flags & ~Fcntl::O_NONBLOCK) if stdout_flags & Fcntl::O_NONBLOCK - result + stderr_flags = @stderr_out.fcntl(Fcntl::F_GETFL) + @stderr_out.fcntl(Fcntl::F_SETFL, stderr_flags & ~Fcntl::O_NONBLOCK) if stderr_flags & Fcntl::O_NONBLOCK + rescue NameError, NotImplementedError, Errno::EINVAL + # Probably on windows, where pipes are blocking by default end end end diff --git a/spec/controllers/admin/dashboard_controller_spec.rb b/spec/controllers/admin/dashboard_controller_spec.rb index 73b50e721..7824854f9 100644 --- a/spec/controllers/admin/dashboard_controller_spec.rb +++ b/spec/controllers/admin/dashboard_controller_spec.rb @@ -3,9 +3,19 @@ require 'rails_helper' describe Admin::DashboardController, type: :controller do + render_views + describe 'GET #index' do - it 'returns 200' do + before do + allow(Admin::SystemCheck).to receive(:perform).and_return([ + Admin::SystemCheck::Message.new(:database_schema_check), + Admin::SystemCheck::Message.new(:rules_check, nil, admin_rules_path), + Admin::SystemCheck::Message.new(:sidekiq_process_check, 'foo, bar'), + ]) sign_in Fabricate(:user, admin: true) + end + + it 'returns 200' do get :index expect(response).to have_http_status(200) diff --git a/spec/mailers/notification_mailer_spec.rb b/spec/mailers/notification_mailer_spec.rb index 3ae106218..9b645bad8 100644 --- a/spec/mailers/notification_mailer_spec.rb +++ b/spec/mailers/notification_mailer_spec.rb @@ -10,12 +10,12 @@ RSpec.describe NotificationMailer, type: :mailer do it 'renders subject localized for the locale of the receiver' do locale = %i(de en).sample receiver.update!(locale: locale) - expect(mail.subject).to eq I18n.t(*args, kwrest.merge(locale: locale)) + expect(mail.subject).to eq I18n.t(*args, **kwrest.merge(locale: locale)) end it 'renders subject localized for the default locale if the locale of the receiver is unavailable' do receiver.update!(locale: nil) - expect(mail.subject).to eq I18n.t(*args, kwrest.merge(locale: I18n.default_locale)) + expect(mail.subject).to eq I18n.t(*args, **kwrest.merge(locale: I18n.default_locale)) end end diff --git a/spec/mailers/user_mailer_spec.rb b/spec/mailers/user_mailer_spec.rb index 6b430b505..9c866788f 100644 --- a/spec/mailers/user_mailer_spec.rb +++ b/spec/mailers/user_mailer_spec.rb @@ -9,12 +9,12 @@ describe UserMailer, type: :mailer do it 'renders subject localized for the locale of the receiver' do locale = I18n.available_locales.sample receiver.update!(locale: locale) - expect(mail.subject).to eq I18n.t(*args, kwrest.merge(locale: locale)) + expect(mail.subject).to eq I18n.t(*args, **kwrest.merge(locale: locale)) end it 'renders subject localized for the default locale if the locale of the receiver is unavailable' do receiver.update!(locale: nil) - expect(mail.subject).to eq I18n.t(*args, kwrest.merge(locale: I18n.default_locale)) + expect(mail.subject).to eq I18n.t(*args, **kwrest.merge(locale: I18n.default_locale)) end end diff --git a/spec/models/session_activation_spec.rb b/spec/models/session_activation_spec.rb index 2aa695037..450dc1399 100644 --- a/spec/models/session_activation_spec.rb +++ b/spec/models/session_activation_spec.rb @@ -74,13 +74,13 @@ RSpec.describe SessionActivation, type: :model do let(:options) { { user: Fabricate(:user), session_id: '1' } } it 'calls create! and purge_old' do - expect(described_class).to receive(:create!).with(options) + expect(described_class).to receive(:create!).with(**options) expect(described_class).to receive(:purge_old) - described_class.activate(options) + described_class.activate(**options) end it 'returns an instance of SessionActivation' do - expect(described_class.activate(options)).to be_kind_of SessionActivation + expect(described_class.activate(**options)).to be_kind_of SessionActivation end end diff --git a/spec/presenters/account_relationships_presenter_spec.rb b/spec/presenters/account_relationships_presenter_spec.rb index f8b048d38..edfbbb354 100644 --- a/spec/presenters/account_relationships_presenter_spec.rb +++ b/spec/presenters/account_relationships_presenter_spec.rb @@ -13,7 +13,7 @@ RSpec.describe AccountRelationshipsPresenter do allow(Account).to receive(:domain_blocking_map).with(account_ids, current_account_id).and_return(default_map) end - let(:presenter) { AccountRelationshipsPresenter.new(account_ids, current_account_id, options) } + let(:presenter) { AccountRelationshipsPresenter.new(account_ids, current_account_id, **options) } let(:current_account_id) { Fabricate(:account).id } let(:account_ids) { [Fabricate(:account).id] } let(:default_map) { { 1 => true } } -- cgit From 74081433d0078784b7c2139f6caaa812740632b2 Mon Sep 17 00:00:00 2001 From: Eugen Rochko Date: Fri, 7 May 2021 14:33:43 +0200 Subject: Change trending hashtags to be affected be reblogs (#16164) If a status with a hashtag becomes very popular, it stands to reason that the hashtag should have a chance at trending Fix no stats being recorded for hashtags that are not allowed to trend, and stop ignoring bots Remove references to hashtags in profile directory from the code and the admin UI --- app/controllers/directories_controller.rb | 10 ------ app/lib/activitypub/activity/announce.rb | 4 +++ app/lib/activitypub/activity/create.rb | 2 +- app/models/account.rb | 11 ++----- app/models/account_tag_stat.rb | 24 -------------- app/models/tag.rb | 38 +++++----------------- app/models/tag_filter.rb | 2 -- app/models/trending_tags.rb | 12 ++++--- app/services/process_hashtags_service.rb | 3 +- app/services/reblog_service.rb | 11 +++++++ app/views/admin/tags/_tag.html.haml | 2 -- app/views/admin/tags/index.html.haml | 9 ++--- app/views/admin/tags/show.html.haml | 13 ++------ config/locales/en.yml | 7 ++-- config/locales/simple_form.en.yml | 2 +- config/routes.rb | 2 -- .../20210502233513_drop_account_tag_stats.rb | 13 ++++++++ db/schema.rb | 10 ------ spec/models/account_tag_stat_spec.rb | 38 ---------------------- spec/models/trending_tags_spec.rb | 6 ++-- 20 files changed, 59 insertions(+), 160 deletions(-) delete mode 100644 app/models/account_tag_stat.rb create mode 100644 db/post_migrate/20210502233513_drop_account_tag_stats.rb delete mode 100644 spec/models/account_tag_stat_spec.rb (limited to 'app/controllers') diff --git a/app/controllers/directories_controller.rb b/app/controllers/directories_controller.rb index f198ad5ba..f28c5b2af 100644 --- a/app/controllers/directories_controller.rb +++ b/app/controllers/directories_controller.rb @@ -6,7 +6,6 @@ class DirectoriesController < ApplicationController before_action :authenticate_user!, if: :whitelist_mode? before_action :require_enabled! before_action :set_instance_presenter - before_action :set_tag, only: :show before_action :set_accounts skip_before_action :require_functional!, unless: :whitelist_mode? @@ -15,23 +14,14 @@ class DirectoriesController < ApplicationController render :index end - def show - render :index - end - private def require_enabled! return not_found unless Setting.profile_directory end - def set_tag - @tag = Tag.discoverable.find_normalized!(params[:id]) - end - def set_accounts @accounts = Account.local.discoverable.by_recent_status.page(params[:page]).per(20).tap do |query| - query.merge!(Account.tagged_with(@tag.id)) if @tag query.merge!(Account.not_excluded_by_account(current_account)) if current_account end end diff --git a/app/lib/activitypub/activity/announce.rb b/app/lib/activitypub/activity/announce.rb index a1081522e..9f778ffb9 100644 --- a/app/lib/activitypub/activity/announce.rb +++ b/app/lib/activitypub/activity/announce.rb @@ -22,6 +22,10 @@ class ActivityPub::Activity::Announce < ActivityPub::Activity visibility: visibility_from_audience ) + original_status.tags.each do |tag| + tag.use!(@account) + end + distribute(@status) end diff --git a/app/lib/activitypub/activity/create.rb b/app/lib/activitypub/activity/create.rb index c7a655d9d..e46361c14 100644 --- a/app/lib/activitypub/activity/create.rb +++ b/app/lib/activitypub/activity/create.rb @@ -164,7 +164,7 @@ class ActivityPub::Activity::Create < ActivityPub::Activity def attach_tags(status) @tags.each do |tag| status.tags << tag - TrendingTags.record_use!(tag, status.account, status.created_at) if status.public_visibility? + tag.use!(@account, status: status, at_time: status.created_at) if status.public_visibility? end @mentions.each do |mention| diff --git a/app/models/account.rb b/app/models/account.rb index a573365de..994459338 100644 --- a/app/models/account.rb +++ b/app/models/account.rb @@ -111,7 +111,6 @@ class Account < ApplicationRecord scope :searchable, -> { without_suspended.where(moved_to_account_id: nil) } scope :discoverable, -> { searchable.without_silenced.where(discoverable: true).left_outer_joins(:account_stat) } scope :followable_by, ->(account) { joins(arel_table.join(Follow.arel_table, Arel::Nodes::OuterJoin).on(arel_table[:id].eq(Follow.arel_table[:target_account_id]).and(Follow.arel_table[:account_id].eq(account.id))).join_sources).where(Follow.arel_table[:id].eq(nil)).joins(arel_table.join(FollowRequest.arel_table, Arel::Nodes::OuterJoin).on(arel_table[:id].eq(FollowRequest.arel_table[:target_account_id]).and(FollowRequest.arel_table[:account_id].eq(account.id))).join_sources).where(FollowRequest.arel_table[:id].eq(nil)) } - scope :tagged_with, ->(tag) { joins(:accounts_tags).where(accounts_tags: { tag_id: tag }) } scope :by_recent_status, -> { order(Arel.sql('(case when account_stats.last_status_at is null then 1 else 0 end) asc, account_stats.last_status_at desc, accounts.id desc')) } scope :by_recent_sign_in, -> { order(Arel.sql('(case when users.current_sign_in_at is null then 1 else 0 end) asc, users.current_sign_in_at desc, accounts.id desc')) } scope :popular, -> { order('account_stats.followers_count desc') } @@ -279,19 +278,13 @@ class Account < ApplicationRecord if hashtags_map.key?(tag.name) hashtags_map.delete(tag.name) else - transaction do - tags.delete(tag) - tag.decrement_count!(:accounts_count) - end + tags.delete(tag) end end # Add hashtags that were so far missing hashtags_map.each_value do |tag| - transaction do - tags << tag - tag.increment_count!(:accounts_count) - end + tags << tag end end diff --git a/app/models/account_tag_stat.rb b/app/models/account_tag_stat.rb deleted file mode 100644 index 3c36c155a..000000000 --- a/app/models/account_tag_stat.rb +++ /dev/null @@ -1,24 +0,0 @@ -# frozen_string_literal: true -# == Schema Information -# -# Table name: account_tag_stats -# -# id :bigint(8) not null, primary key -# tag_id :bigint(8) not null -# accounts_count :bigint(8) default(0), not null -# hidden :boolean default(FALSE), not null -# created_at :datetime not null -# updated_at :datetime not null -# - -class AccountTagStat < ApplicationRecord - belongs_to :tag, inverse_of: :account_tag_stat - - def increment_count!(key) - update(key => public_send(key) + 1) - end - - def decrement_count!(key) - update(key => [public_send(key) - 1, 0].max) - end -end diff --git a/app/models/tag.rb b/app/models/tag.rb index efffc7eee..735c30608 100644 --- a/app/models/tag.rb +++ b/app/models/tag.rb @@ -20,10 +20,8 @@ class Tag < ApplicationRecord has_and_belongs_to_many :statuses has_and_belongs_to_many :accounts - has_and_belongs_to_many :sample_accounts, -> { local.discoverable.popular.limit(3) }, class_name: 'Account' has_many :featured_tags, dependent: :destroy, inverse_of: :tag - has_one :account_tag_stat, dependent: :destroy HASHTAG_SEPARATORS = "_\u00B7\u200c" HASHTAG_NAME_RE = "([[:word:]_][[:word:]#{HASHTAG_SEPARATORS}]*[[:alpha:]#{HASHTAG_SEPARATORS}][[:word:]#{HASHTAG_SEPARATORS}]*[[:word:]_])|([[:word:]_]*[[:alpha:]][[:word:]_]*)" @@ -38,29 +36,11 @@ class Tag < ApplicationRecord scope :usable, -> { where(usable: [true, nil]) } scope :listable, -> { where(listable: [true, nil]) } scope :trendable, -> { Setting.trendable_by_default ? where(trendable: [true, nil]) : where(trendable: true) } - scope :discoverable, -> { listable.joins(:account_tag_stat).where(AccountTagStat.arel_table[:accounts_count].gt(0)).order(Arel.sql('account_tag_stats.accounts_count desc')) } scope :recently_used, ->(account) { joins(:statuses).where(statuses: { id: account.statuses.select(:id).limit(1000) }).group(:id).order(Arel.sql('count(*) desc')) } - # Search with case-sensitive to use B-tree index. - scope :matches_name, ->(term) { where(arel_table[:name].lower.matches(arel_table.lower("#{sanitize_sql_like(Tag.normalize(term))}%"), nil, true)) } - - delegate :accounts_count, - :accounts_count=, - :increment_count!, - :decrement_count!, - to: :account_tag_stat - - after_save :save_account_tag_stat + scope :matches_name, ->(term) { where(arel_table[:name].lower.matches(arel_table.lower("#{sanitize_sql_like(Tag.normalize(term))}%"), nil, true)) } # Search with case-sensitive to use B-tree index update_index('tags#tag', :self) - def account_tag_stat - super || build_account_tag_stat - end - - def cached_sample_accounts - Rails.cache.fetch("#{cache_key}/sample_accounts", expires_in: 12.hours) { sample_accounts } - end - def to_param name end @@ -95,6 +75,10 @@ class Tag < ApplicationRecord requested_review_at.present? end + def use!(account, status: nil, at_time: Time.now.utc) + TrendingTags.record_use!(self, account, status: status, at_time: at_time) + end + def trending? TrendingTags.trending?(self) end @@ -127,9 +111,10 @@ class Tag < ApplicationRecord end def search_for(term, limit = 5, offset = 0, options = {}) - striped_term = term.strip - query = Tag.listable.matches_name(striped_term) - query = query.merge(matching_name(striped_term).or(where.not(reviewed_at: nil))) if options[:exclude_unreviewed] + stripped_term = term.strip + + query = Tag.listable.matches_name(stripped_term) + query = query.merge(matching_name(stripped_term).or(where.not(reviewed_at: nil))) if options[:exclude_unreviewed] query.order(Arel.sql('length(name) ASC, name ASC')) .limit(limit) @@ -161,11 +146,6 @@ class Tag < ApplicationRecord private - def save_account_tag_stat - return unless account_tag_stat&.changed? - account_tag_stat.save - end - def validate_name_change errors.add(:name, I18n.t('tags.does_not_match_previous_name')) unless name_was.mb_chars.casecmp(name.mb_chars).zero? end diff --git a/app/models/tag_filter.rb b/app/models/tag_filter.rb index a9ff5b703..85bfcbea5 100644 --- a/app/models/tag_filter.rb +++ b/app/models/tag_filter.rb @@ -33,8 +33,6 @@ class TagFilter def scope_for(key, value) case key.to_s - when 'directory' - Tag.discoverable when 'reviewed' Tag.reviewed.order(reviewed_at: :desc) when 'unreviewed' diff --git a/app/models/trending_tags.rb b/app/models/trending_tags.rb index 9c2aa0ee8..31890b082 100644 --- a/app/models/trending_tags.rb +++ b/app/models/trending_tags.rb @@ -13,19 +13,23 @@ class TrendingTags class << self include Redisable - def record_use!(tag, account, at_time = Time.now.utc) - return if account.silenced? || account.bot? || !tag.usable? || !(tag.trendable? || tag.requires_review?) + def record_use!(tag, account, status: nil, at_time: Time.now.utc) + return unless tag.usable? && !account.silenced? + # Even if a tag is not allowed to trend, we still need to + # record the stats since they can be displayed in other places increment_historical_use!(tag.id, at_time) increment_unique_use!(tag.id, account.id, at_time) increment_use!(tag.id, at_time) - tag.update(last_status_at: Time.now.utc) if tag.last_status_at.nil? || tag.last_status_at < 12.hours.ago + # Only update when the tag was last used once every 12 hours + # and only if a status is given (lets use ignore reblogs) + tag.update(last_status_at: at_time) if status.present? && (tag.last_status_at.nil? || (tag.last_status_at < at_time && tag.last_status_at < 12.hours.ago)) end def update!(at_time = Time.now.utc) tag_ids = redis.smembers("#{KEY}:used:#{at_time.beginning_of_day.to_i}") + redis.zrange(KEY, 0, -1) - tags = Tag.where(id: tag_ids.uniq) + tags = Tag.trendable.where(id: tag_ids.uniq) # First pass to calculate scores and update the set diff --git a/app/services/process_hashtags_service.rb b/app/services/process_hashtags_service.rb index e8e139b05..c42b79db8 100644 --- a/app/services/process_hashtags_service.rb +++ b/app/services/process_hashtags_service.rb @@ -8,8 +8,7 @@ class ProcessHashtagsService < BaseService Tag.find_or_create_by_names(tags) do |tag| status.tags << tag records << tag - - TrendingTags.record_use!(tag, status.account, status.created_at) if status.public_visibility? + tag.use!(status.account, status: status, at_time: status.created_at) if status.public_visibility? end return unless status.distributable? diff --git a/app/services/reblog_service.rb b/app/services/reblog_service.rb index 5032397b3..744bdf567 100644 --- a/app/services/reblog_service.rb +++ b/app/services/reblog_service.rb @@ -35,6 +35,7 @@ class ReblogService < BaseService create_notification(reblog) bump_potential_friendship(account, reblog) + record_use(account, reblog) reblog end @@ -59,6 +60,16 @@ class ReblogService < BaseService PotentialFriendshipTracker.record(account.id, reblog.reblog.account_id, :reblog) end + def record_use(account, reblog) + return unless reblog.public_visibility? + + original_status = reblog.reblog + + original_status.tags.each do |tag| + tag.use!(account) + end + end + def build_json(reblog) Oj.dump(serialize_payload(ActivityPub::ActivityPresenter.from_status(reblog), ActivityPub::ActivitySerializer, signer: reblog.account)) end diff --git a/app/views/admin/tags/_tag.html.haml b/app/views/admin/tags/_tag.html.haml index 287d28e53..adf4ca7b2 100644 --- a/app/views/admin/tags/_tag.html.haml +++ b/app/views/admin/tags/_tag.html.haml @@ -10,8 +10,6 @@ = tag.name %small - = t('admin.tags.in_directory', count: tag.accounts_count) - • = t('admin.tags.unique_uses_today', count: tag.history.first[:accounts]) - if tag.trending? diff --git a/app/views/admin/tags/index.html.haml b/app/views/admin/tags/index.html.haml index d7719d45d..d78f3c6d1 100644 --- a/app/views/admin/tags/index.html.haml +++ b/app/views/admin/tags/index.html.haml @@ -5,12 +5,6 @@ = javascript_pack_tag 'admin', async: true, crossorigin: 'anonymous' .filters - .filter-subset - %strong= t('admin.tags.context') - %ul - %li= filter_link_to t('generic.all'), directory: nil - %li= filter_link_to t('admin.tags.directory'), directory: '1' - .filter-subset %strong= t('admin.tags.review') %ul @@ -23,8 +17,9 @@ %strong= t('generic.order_by') %ul %li= filter_link_to t('admin.tags.most_recent'), popular: nil, active: nil - %li= filter_link_to t('admin.tags.most_popular'), popular: '1', active: nil %li= filter_link_to t('admin.tags.last_active'), active: '1', popular: nil + %li= filter_link_to t('admin.tags.most_popular'), popular: '1', active: nil + = form_tag admin_tags_url, method: 'GET', class: 'simple_form' do .fields-group diff --git a/app/views/admin/tags/show.html.haml b/app/views/admin/tags/show.html.haml index c9a147587..c4caffda1 100644 --- a/app/views/admin/tags/show.html.haml +++ b/app/views/admin/tags/show.html.haml @@ -10,15 +10,6 @@ %div .dashboard__counters__num= number_with_delimiter @accounts_week .dashboard__counters__label= t 'admin.tags.accounts_week' - %div - - if @tag.accounts_count > 0 - = link_to explore_hashtag_path(@tag) do - .dashboard__counters__num= number_with_delimiter @tag.accounts_count - .dashboard__counters__label= t 'admin.tags.directory' - - else - %div - .dashboard__counters__num= number_with_delimiter @tag.accounts_count - .dashboard__counters__label= t 'admin.tags.directory' %hr.spacer/ @@ -30,8 +21,8 @@ .fields-group = f.input :usable, as: :boolean, wrapper: :with_label - = f.input :trendable, as: :boolean, wrapper: :with_label, disabled: !Setting.trends - = f.input :listable, as: :boolean, wrapper: :with_label, disabled: !Setting.profile_directory + = f.input :trendable, as: :boolean, wrapper: :with_label + = f.input :listable, as: :boolean, wrapper: :with_label .actions = f.button :button, t('generic.save_changes'), type: :submit diff --git a/config/locales/en.yml b/config/locales/en.yml index bfa489817..d8ad5bd84 100644 --- a/config/locales/en.yml +++ b/config/locales/en.yml @@ -698,12 +698,9 @@ en: accounts_today: Unique uses today accounts_week: Unique uses this week breakdown: Breakdown of today's usage by source - context: Context - directory: In directory - in_directory: "%{count} in directory" - last_active: Last active + last_active: Recently used most_popular: Most popular - most_recent: Most recent + most_recent: Recently created name: Hashtag review: Review status reviewed: Reviewed diff --git a/config/locales/simple_form.en.yml b/config/locales/simple_form.en.yml index 8ff880ebc..c4388ffc5 100644 --- a/config/locales/simple_form.en.yml +++ b/config/locales/simple_form.en.yml @@ -208,7 +208,7 @@ en: rule: text: Rule tag: - listable: Allow this hashtag to appear in searches and on the profile directory + listable: Allow this hashtag to appear in searches and suggestions name: Hashtag trendable: Allow this hashtag to appear under trends usable: Allow posts to use this hashtag diff --git a/config/routes.rb b/config/routes.rb index 8ca7fccdd..2373d8a51 100644 --- a/config/routes.rb +++ b/config/routes.rb @@ -97,8 +97,6 @@ Rails.application.routes.draw do post '/interact/:id', to: 'remote_interaction#create' get '/explore', to: 'directories#index', as: :explore - get '/explore/:id', to: 'directories#show', as: :explore_hashtag - get '/settings', to: redirect('/settings/profile') namespace :settings do diff --git a/db/post_migrate/20210502233513_drop_account_tag_stats.rb b/db/post_migrate/20210502233513_drop_account_tag_stats.rb new file mode 100644 index 000000000..80adadcab --- /dev/null +++ b/db/post_migrate/20210502233513_drop_account_tag_stats.rb @@ -0,0 +1,13 @@ +# frozen_string_literal: true + +class DropAccountTagStats < ActiveRecord::Migration[5.2] + disable_ddl_transaction! + + def up + drop_table :account_tag_stats + end + + def down + raise ActiveRecord::IrreversibleMigration + end +end diff --git a/db/schema.rb b/db/schema.rb index 88e906079..19b1afe00 100644 --- a/db/schema.rb +++ b/db/schema.rb @@ -115,15 +115,6 @@ ActiveRecord::Schema.define(version: 2021_05_05_174616) do t.index ["account_id"], name: "index_account_stats_on_account_id", unique: true end - create_table "account_tag_stats", force: :cascade do |t| - t.bigint "tag_id", null: false - t.bigint "accounts_count", default: 0, null: false - t.boolean "hidden", default: false, null: false - t.datetime "created_at", null: false - t.datetime "updated_at", null: false - t.index ["tag_id"], name: "index_account_tag_stats_on_tag_id", unique: true - end - create_table "account_warning_presets", force: :cascade do |t| t.text "text", default: "", null: false t.datetime "created_at", null: false @@ -985,7 +976,6 @@ ActiveRecord::Schema.define(version: 2021_05_05_174616) do add_foreign_key "account_pins", "accounts", column: "target_account_id", on_delete: :cascade add_foreign_key "account_pins", "accounts", on_delete: :cascade add_foreign_key "account_stats", "accounts", on_delete: :cascade - add_foreign_key "account_tag_stats", "tags", on_delete: :cascade add_foreign_key "account_warnings", "accounts", column: "target_account_id", on_delete: :cascade add_foreign_key "account_warnings", "accounts", on_delete: :nullify add_foreign_key "accounts", "accounts", column: "moved_to_account_id", on_delete: :nullify diff --git a/spec/models/account_tag_stat_spec.rb b/spec/models/account_tag_stat_spec.rb deleted file mode 100644 index 6d3057f35..000000000 --- a/spec/models/account_tag_stat_spec.rb +++ /dev/null @@ -1,38 +0,0 @@ -# frozen_string_literal: true - -require 'rails_helper' - -RSpec.describe AccountTagStat, type: :model do - key = 'accounts_count' - let(:account_tag_stat) { Fabricate(:tag).account_tag_stat } - - describe '#increment_count!' do - it 'calls #update' do - args = { key => account_tag_stat.public_send(key) + 1 } - expect(account_tag_stat).to receive(:update).with(args) - account_tag_stat.increment_count!(key) - end - - it 'increments value by 1' do - expect do - account_tag_stat.increment_count!(key) - end.to change { account_tag_stat.accounts_count }.by(1) - end - end - - describe '#decrement_count!' do - it 'calls #update' do - args = { key => [account_tag_stat.public_send(key) - 1, 0].max } - expect(account_tag_stat).to receive(:update).with(args) - account_tag_stat.decrement_count!(key) - end - - it 'decrements value by 1' do - account_tag_stat.update(key => 1) - - expect do - account_tag_stat.decrement_count!(key) - end.to change { account_tag_stat.accounts_count }.by(-1) - end - end -end diff --git a/spec/models/trending_tags_spec.rb b/spec/models/trending_tags_spec.rb index b6122c994..dfbc7d6f8 100644 --- a/spec/models/trending_tags_spec.rb +++ b/spec/models/trending_tags_spec.rb @@ -7,9 +7,9 @@ RSpec.describe TrendingTags do describe '.update!' do let!(:at_time) { Time.now.utc } - let!(:tag1) { Fabricate(:tag, name: 'Catstodon') } - let!(:tag2) { Fabricate(:tag, name: 'DogsOfMastodon') } - let!(:tag3) { Fabricate(:tag, name: 'OCs') } + let!(:tag1) { Fabricate(:tag, name: 'Catstodon', trendable: true) } + let!(:tag2) { Fabricate(:tag, name: 'DogsOfMastodon', trendable: true) } + let!(:tag3) { Fabricate(:tag, name: 'OCs', trendable: true) } before do allow(Redis.current).to receive(:pfcount) do |key| -- cgit