From 0cc77263fc4aba44a3c1277ffe617b89d083cfce Mon Sep 17 00:00:00 2001 From: Claire Date: Thu, 17 Nov 2022 10:52:51 +0100 Subject: Change batch account suspension to create a strike (#20897) --- app/models/form/account_batch.rb | 4 ++++ 1 file changed, 4 insertions(+) (limited to 'app/models') diff --git a/app/models/form/account_batch.rb b/app/models/form/account_batch.rb index 5cfcf7205..473622edf 100644 --- a/app/models/form/account_batch.rb +++ b/app/models/form/account_batch.rb @@ -115,6 +115,10 @@ class Form::AccountBatch authorize(account, :suspend?) log_action(:suspend, account) account.suspend!(origin: :local) + account.strikes.create!( + account: current_account, + action: :suspend + ) Admin::SuspensionWorker.perform_async(account.id) end -- cgit From 00b2720ef08e91bee88cd24da7eb2ba836a7a10f Mon Sep 17 00:00:00 2001 From: Claire Date: Thu, 17 Nov 2022 10:55:23 +0100 Subject: Change automatic post deletion configuration to be accessible to redirected users (#20774) Fixes #20550 --- app/controllers/statuses_cleanup_controller.rb | 4 ++++ app/models/user.rb | 6 +++++- config/navigation.rb | 2 +- 3 files changed, 10 insertions(+), 2 deletions(-) (limited to 'app/models') diff --git a/app/controllers/statuses_cleanup_controller.rb b/app/controllers/statuses_cleanup_controller.rb index be234cdcb..e912967fd 100644 --- a/app/controllers/statuses_cleanup_controller.rb +++ b/app/controllers/statuses_cleanup_controller.rb @@ -19,6 +19,10 @@ class StatusesCleanupController < ApplicationController # Do nothing end + def require_functional! + redirect_to edit_user_registration_path unless current_user.functional_or_moved? + end + private def set_policy diff --git a/app/models/user.rb b/app/models/user.rb index 6d566b1c2..3d0298927 100644 --- a/app/models/user.rb +++ b/app/models/user.rb @@ -237,7 +237,11 @@ class User < ApplicationRecord end def functional? - confirmed? && approved? && !disabled? && !account.suspended? && !account.memorial? && account.moved_to_account_id.nil? + functional_or_moved? && account.moved_to_account_id.nil? + end + + def functional_or_moved? + confirmed? && approved? && !disabled? && !account.suspended? && !account.memorial? end def unconfirmed? diff --git a/config/navigation.rb b/config/navigation.rb index e901fb932..30817d025 100644 --- a/config/navigation.rb +++ b/config/navigation.rb @@ -17,7 +17,7 @@ SimpleNavigation::Configuration.run do |navigation| n.item :relationships, safe_join([fa_icon('users fw'), t('settings.relationships')]), relationships_path, if: -> { current_user.functional? } n.item :filters, safe_join([fa_icon('filter fw'), t('filters.index.title')]), filters_path, highlights_on: %r{/filters}, if: -> { current_user.functional? } - n.item :statuses_cleanup, safe_join([fa_icon('history fw'), t('settings.statuses_cleanup')]), statuses_cleanup_path, if: -> { current_user.functional? } + n.item :statuses_cleanup, safe_join([fa_icon('history fw'), t('settings.statuses_cleanup')]), statuses_cleanup_path, if: -> { current_user.functional_or_moved? } n.item :security, safe_join([fa_icon('lock fw'), t('settings.account')]), edit_user_registration_path do |s| s.item :password, safe_join([fa_icon('lock fw'), t('settings.account_settings')]), edit_user_registration_path, highlights_on: %r{/auth/edit|/settings/delete|/settings/migration|/settings/aliases|/settings/login_activities|^/disputes} -- cgit From c373148b3d43056c242fbb891510f1f841ca2f45 Mon Sep 17 00:00:00 2001 From: lenore gilbert Date: Thu, 17 Nov 2022 03:05:09 -0700 Subject: Support for import/export of instance-level domain blocks/allows for 4.x w/ additional fixes (#20597) * Allow import/export of instance-level domain blocks/allows (#1754) * Allow import/export of instance-level domain blocks/allows. Fixes #15095 * Pacify circleci * Address simple code review feedback * Add headers to exported CSV * Extract common import/export functionality to AdminExportControllerConcern * Add additional fields to instance-blocked domain export * Address review feedback * Split instance domain block/allow import/export into separate pages/controllers * Address code review feedback * Pacify DeepSource * Work around Paperclip::HasAttachmentFile for Rails 6 * Fix deprecated API warning in export tests * Remove after_commit workaround (cherry picked from commit 94e98864e39c010635e839fea984f2b4893bef1a) * Add confirmation page when importing blocked domains (#1773) * Move glitch-soc-specific strings to glitch-soc-specific locale files * Add confirmation page when importing blocked domains (cherry picked from commit b91196f4b73fff91997b8077619ae25b6d04a59e) * Fix authorization check in domain blocks controller (cherry picked from commit 75279377583c6e2aa04cc8d7380c593979630b38) * Fix error strings for domain blocks and email-domain blocks Corrected issue with non-error message used for Mastodon:NotPermittedError in Domain Blocks Corrected issue Domain Blocks using the Email Domain Blocks message on ActionContoller::ParameterMissing Corrected issue with Email Domain Blocks using the not_permitted string from "custom emojii's" * Ran i18n-tasks normalize to address test failure * Removed unused admin.export_domain_blocks.not_permitted string Removing unused string as indicated by Check i18n * Fix tests (cherry picked from commit 9094c2f52c24e1c00b594e7c11cd00e4a07eb431) * Fix domain block export not exporting blocks with only media rejection (cherry picked from commit 26ff48ee48a5c03a2a4b0bd03fd322529e6bd960) * Fix various issues with domain block import - stop using Paperclip for processing domain allow/block imports - stop leaving temporary files - better error handling - assume CSV files are UTF-8-encoded (cherry picked from commit cad824d8f501b95377e4f0a957e5a00d517a1902) Co-authored-by: Levi Bard Co-authored-by: Claire --- app/controllers/admin/domain_blocks_controller.rb | 22 +++++++ .../admin/email_domain_blocks_controller.rb | 2 +- .../admin/export_domain_allows_controller.rb | 60 ++++++++++++++++++ .../admin/export_domain_blocks_controller.rb | 71 ++++++++++++++++++++++ .../concerns/admin_export_controller_concern.rb | 39 ++++++++++++ app/javascript/packs/admin.js | 6 ++ app/models/admin/import.rb | 32 ++++++++++ app/models/domain_allow.rb | 4 ++ app/models/domain_block.rb | 1 + app/models/form/domain_block_batch.rb | 35 +++++++++++ app/views/admin/export_domain_allows/new.html.haml | 10 +++ .../export_domain_blocks/_domain_block.html.haml | 27 ++++++++ .../admin/export_domain_blocks/import.html.haml | 21 +++++++ app/views/admin/export_domain_blocks/new.html.haml | 10 +++ app/views/admin/instances/index.html.haml | 4 ++ config/locales/en.yml | 21 +++++++ config/routes.rb | 21 ++++++- .../admin/domain_allows_controller_spec.rb | 48 +++++++++++++++ .../admin/domain_blocks_controller_spec.rb | 21 +++++++ .../admin/export_domain_allows_controller_spec.rb | 42 +++++++++++++ .../admin/export_domain_blocks_controller_spec.rb | 35 +++++++++++ spec/fixtures/files/domain_allows.csv | 3 + spec/fixtures/files/domain_blocks.csv | 4 ++ 23 files changed, 537 insertions(+), 2 deletions(-) create mode 100644 app/controllers/admin/export_domain_allows_controller.rb create mode 100644 app/controllers/admin/export_domain_blocks_controller.rb create mode 100644 app/controllers/concerns/admin_export_controller_concern.rb create mode 100644 app/models/admin/import.rb create mode 100644 app/models/form/domain_block_batch.rb create mode 100644 app/views/admin/export_domain_allows/new.html.haml create mode 100644 app/views/admin/export_domain_blocks/_domain_block.html.haml create mode 100644 app/views/admin/export_domain_blocks/import.html.haml create mode 100644 app/views/admin/export_domain_blocks/new.html.haml create mode 100644 spec/controllers/admin/domain_allows_controller_spec.rb create mode 100644 spec/controllers/admin/export_domain_allows_controller_spec.rb create mode 100644 spec/controllers/admin/export_domain_blocks_controller_spec.rb create mode 100644 spec/fixtures/files/domain_allows.csv create mode 100644 spec/fixtures/files/domain_blocks.csv (limited to 'app/models') diff --git a/app/controllers/admin/domain_blocks_controller.rb b/app/controllers/admin/domain_blocks_controller.rb index 16defc1ea..e79f7a43e 100644 --- a/app/controllers/admin/domain_blocks_controller.rb +++ b/app/controllers/admin/domain_blocks_controller.rb @@ -4,6 +4,18 @@ module Admin class DomainBlocksController < BaseController before_action :set_domain_block, only: [:show, :destroy, :edit, :update] + def batch + authorize :domain_block, :create? + @form = Form::DomainBlockBatch.new(form_domain_block_batch_params.merge(current_account: current_account, action: action_from_button)) + @form.save + rescue ActionController::ParameterMissing + flash[:alert] = I18n.t('admin.domain_blocks.no_domain_block_selected') + rescue Mastodon::NotPermittedError + flash[:alert] = I18n.t('admin.domain_blocks.not_permitted') + else + redirect_to admin_instances_path(limited: '1'), notice: I18n.t('admin.domain_blocks.created_msg') + end + def new authorize :domain_block, :create? @domain_block = DomainBlock.new(domain: params[:_domain]) @@ -76,5 +88,15 @@ module Admin def resource_params params.require(:domain_block).permit(:domain, :severity, :reject_media, :reject_reports, :private_comment, :public_comment, :obfuscate) end + + def form_domain_block_batch_params + params.require(:form_domain_block_batch).permit(domain_blocks_attributes: [:enabled, :domain, :severity, :reject_media, :reject_reports, :private_comment, :public_comment, :obfuscate]) + end + + def action_from_button + if params[:save] + 'save' + end + end end end diff --git a/app/controllers/admin/email_domain_blocks_controller.rb b/app/controllers/admin/email_domain_blocks_controller.rb index 593457b94..a0a43de19 100644 --- a/app/controllers/admin/email_domain_blocks_controller.rb +++ b/app/controllers/admin/email_domain_blocks_controller.rb @@ -19,7 +19,7 @@ module Admin rescue ActionController::ParameterMissing flash[:alert] = I18n.t('admin.email_domain_blocks.no_email_domain_block_selected') rescue Mastodon::NotPermittedError - flash[:alert] = I18n.t('admin.custom_emojis.not_permitted') + flash[:alert] = I18n.t('admin.email_domain_blocks.not_permitted') ensure redirect_to admin_email_domain_blocks_path end diff --git a/app/controllers/admin/export_domain_allows_controller.rb b/app/controllers/admin/export_domain_allows_controller.rb new file mode 100644 index 000000000..57fb12c62 --- /dev/null +++ b/app/controllers/admin/export_domain_allows_controller.rb @@ -0,0 +1,60 @@ +# frozen_string_literal: true + +require 'csv' + +module Admin + class ExportDomainAllowsController < BaseController + include AdminExportControllerConcern + + before_action :set_dummy_import!, only: [:new] + + def new + authorize :domain_allow, :create? + end + + def export + authorize :instance, :index? + send_export_file + end + + def import + authorize :domain_allow, :create? + begin + @import = Admin::Import.new(import_params) + return render :new unless @import.validate + + parse_import_data!(export_headers) + + @data.take(Admin::Import::ROWS_PROCESSING_LIMIT).each do |row| + domain = row['#domain'].strip + next if DomainAllow.allowed?(domain) + + domain_allow = DomainAllow.new(domain: domain) + log_action :create, domain_allow if domain_allow.save + end + flash[:notice] = I18n.t('admin.domain_allows.created_msg') + rescue ActionController::ParameterMissing + flash[:error] = I18n.t('admin.export_domain_allows.no_file') + end + redirect_to admin_instances_path + end + + private + + def export_filename + 'domain_allows.csv' + end + + def export_headers + %w(#domain) + end + + def export_data + CSV.generate(headers: export_headers, write_headers: true) do |content| + DomainAllow.allowed_domains.each do |instance| + content << [instance.domain] + end + end + end + end +end diff --git a/app/controllers/admin/export_domain_blocks_controller.rb b/app/controllers/admin/export_domain_blocks_controller.rb new file mode 100644 index 000000000..fb0cd05d2 --- /dev/null +++ b/app/controllers/admin/export_domain_blocks_controller.rb @@ -0,0 +1,71 @@ +# frozen_string_literal: true + +require 'csv' + +module Admin + class ExportDomainBlocksController < BaseController + include AdminExportControllerConcern + + before_action :set_dummy_import!, only: [:new] + + def new + authorize :domain_block, :create? + end + + def export + authorize :instance, :index? + send_export_file + end + + def import + authorize :domain_block, :create? + + @import = Admin::Import.new(import_params) + return render :new unless @import.validate + + parse_import_data!(export_headers) + + @global_private_comment = I18n.t('admin.export_domain_blocks.import.private_comment_template', source: @import.data_file_name, date: I18n.l(Time.now.utc)) + + @form = Form::DomainBlockBatch.new + @domain_blocks = @data.take(Admin::Import::ROWS_PROCESSING_LIMIT).filter_map do |row| + domain = row['#domain'].strip + next if DomainBlock.rule_for(domain).present? + + domain_block = DomainBlock.new(domain: domain, + severity: row['#severity'].strip, + reject_media: row['#reject_media'].strip, + reject_reports: row['#reject_reports'].strip, + private_comment: @global_private_comment, + public_comment: row['#public_comment']&.strip, + obfuscate: row['#obfuscate'].strip) + + domain_block if domain_block.valid? + end + + @warning_domains = Instance.where(domain: @domain_blocks.map(&:domain)).where('EXISTS (SELECT 1 FROM follows JOIN accounts ON follows.account_id = accounts.id OR follows.target_account_id = accounts.id WHERE accounts.domain = instances.domain)').pluck(:domain) + rescue ActionController::ParameterMissing + flash.now[:alert] = I18n.t('admin.export_domain_blocks.no_file') + set_dummy_import! + render :new + end + + private + + def export_filename + 'domain_blocks.csv' + end + + def export_headers + %w(#domain #severity #reject_media #reject_reports #public_comment #obfuscate) + end + + def export_data + CSV.generate(headers: export_headers, write_headers: true) do |content| + DomainBlock.with_limitations.each do |instance| + content << [instance.domain, instance.severity, instance.reject_media, instance.reject_reports, instance.public_comment, instance.obfuscate] + end + end + end + end +end diff --git a/app/controllers/concerns/admin_export_controller_concern.rb b/app/controllers/concerns/admin_export_controller_concern.rb new file mode 100644 index 000000000..b40c76557 --- /dev/null +++ b/app/controllers/concerns/admin_export_controller_concern.rb @@ -0,0 +1,39 @@ +# frozen_string_literal: true + +module AdminExportControllerConcern + extend ActiveSupport::Concern + + private + + def send_export_file + respond_to do |format| + format.csv { send_data export_data, filename: export_filename } + end + end + + def export_data + raise 'Override in controller' + end + + def export_filename + raise 'Override in controller' + end + + def set_dummy_import! + @import = Admin::Import.new + end + + def import_params + params.require(:admin_import).permit(:data) + end + + def import_data_path + params[:admin_import][:data].path + end + + def parse_import_data!(default_headers) + data = CSV.read(import_data_path, headers: true, encoding: 'UTF-8') + data = CSV.read(import_data_path, headers: default_headers, encoding: 'UTF-8') unless data.headers&.first&.strip&.include?(default_headers[0]) + @data = data.reject(&:blank?) + end +end diff --git a/app/javascript/packs/admin.js b/app/javascript/packs/admin.js index de86e0e11..4e817129d 100644 --- a/app/javascript/packs/admin.js +++ b/app/javascript/packs/admin.js @@ -185,6 +185,12 @@ ready(() => { const registrationMode = document.getElementById('form_admin_settings_registrations_mode'); if (registrationMode) onChangeRegistrationMode(registrationMode); + const checkAllElement = document.querySelector('#batch_checkbox_all'); + if (checkAllElement) { + checkAllElement.checked = [].every.call(document.querySelectorAll(batchCheckboxClassName), (content) => content.checked); + checkAllElement.indeterminate = !checkAllElement.checked && [].some.call(document.querySelectorAll(batchCheckboxClassName), (content) => content.checked); + } + document.querySelector('a#add-instance-button')?.addEventListener('click', (e) => { const domain = document.getElementById('by_domain')?.value; diff --git a/app/models/admin/import.rb b/app/models/admin/import.rb new file mode 100644 index 000000000..79c0722d5 --- /dev/null +++ b/app/models/admin/import.rb @@ -0,0 +1,32 @@ +# frozen_string_literal: true + +# A non-activerecord helper class for csv upload +class Admin::Import + include ActiveModel::Model + + ROWS_PROCESSING_LIMIT = 20_000 + + attr_accessor :data + + validates :data, presence: true + validate :validate_data + + def data_file_name + data.original_filename + end + + private + + def validate_data + return if data.blank? + + csv_data = CSV.read(data.path, encoding: 'UTF-8') + + row_count = csv_data.size + row_count -= 1 if csv_data.first&.first == '#domain' + + errors.add(:data, I18n.t('imports.errors.over_rows_processing_limit', count: ROWS_PROCESSING_LIMIT)) if row_count > ROWS_PROCESSING_LIMIT + rescue CSV::MalformedCSVError => e + errors.add(:data, I18n.t('imports.errors.invalid_csv_file', error: e.message)) + end +end diff --git a/app/models/domain_allow.rb b/app/models/domain_allow.rb index 65f494fed..9e746b915 100644 --- a/app/models/domain_allow.rb +++ b/app/models/domain_allow.rb @@ -28,6 +28,10 @@ class DomainAllow < ApplicationRecord !rule_for(domain).nil? end + def allowed_domains + select(:domain) + end + def rule_for(domain) return if domain.blank? diff --git a/app/models/domain_block.rb b/app/models/domain_block.rb index ad1dc2a38..8e298ac9d 100644 --- a/app/models/domain_block.rb +++ b/app/models/domain_block.rb @@ -29,6 +29,7 @@ class DomainBlock < ApplicationRecord scope :matches_domain, ->(value) { where(arel_table[:domain].matches("%#{value}%")) } scope :with_user_facing_limitations, -> { where(severity: [:silence, :suspend]) } + scope :with_limitations, -> { where(severity: [:silence, :suspend]).or(where(reject_media: true)) } scope :by_severity, -> { order(Arel.sql('(CASE severity WHEN 0 THEN 1 WHEN 1 THEN 2 WHEN 2 THEN 0 END), domain')) } def to_log_human_identifier diff --git a/app/models/form/domain_block_batch.rb b/app/models/form/domain_block_batch.rb new file mode 100644 index 000000000..39012df51 --- /dev/null +++ b/app/models/form/domain_block_batch.rb @@ -0,0 +1,35 @@ +# frozen_string_literal: true + +class Form::DomainBlockBatch + include ActiveModel::Model + include Authorization + include AccountableConcern + + attr_accessor :domain_blocks_attributes, :action, :current_account + + def save + case action + when 'save' + save! + end + end + + private + + def domain_blocks + @domain_blocks ||= domain_blocks_attributes.values.filter_map do |attributes| + DomainBlock.new(attributes.without('enabled')) if ActiveModel::Type::Boolean.new.cast(attributes['enabled']) + end + end + + def save! + domain_blocks.each do |domain_block| + authorize(domain_block, :create?) + next if DomainBlock.rule_for(domain_block.domain).present? + + domain_block.save! + DomainBlockWorker.perform_async(domain_block.id) + log_action :create, domain_block + end + end +end diff --git a/app/views/admin/export_domain_allows/new.html.haml b/app/views/admin/export_domain_allows/new.html.haml new file mode 100644 index 000000000..dc0cf8c52 --- /dev/null +++ b/app/views/admin/export_domain_allows/new.html.haml @@ -0,0 +1,10 @@ +- content_for :page_title do + = t('.title') + += simple_form_for @import, url: import_admin_export_domain_allows_path, html: { multipart: true } do |f| + .fields-row + .fields-group.fields-row__column.fields-row__column-6 + = f.input :data, wrapper: :with_block_label, hint: t('simple_form.hints.imports.data'), as: :file + + .actions + = f.button :button, t('imports.upload'), type: :submit diff --git a/app/views/admin/export_domain_blocks/_domain_block.html.haml b/app/views/admin/export_domain_blocks/_domain_block.html.haml new file mode 100644 index 000000000..5d4b6c4d0 --- /dev/null +++ b/app/views/admin/export_domain_blocks/_domain_block.html.haml @@ -0,0 +1,27 @@ +- existing_relationships ||= false + +.batch-table__row{ class: [existing_relationships && 'batch-table__row--attention'] } + %label.batch-table__row__select.batch-table__row__select--aligned.batch-checkbox + = f.check_box :enabled, checked: !existing_relationships + .batch-table__row__content.pending-account + .pending-account__header + %strong + = f.object.domain + = f.hidden_field :domain + = f.hidden_field :severity + = f.hidden_field :reject_media + = f.hidden_field :reject_reports + = f.hidden_field :obfuscate + = f.hidden_field :private_comment + = f.hidden_field :public_comment + + %br/ + + = f.object.policies.map { |policy| t(policy, scope: 'admin.instances.content_policies.policies') }.join(' • ') + - if f.object.public_comment.present? + • + = f.object.public_comment + - if existing_relationships + • + = fa_icon 'warning fw' + = t('admin.export_domain_blocks.import.existing_relationships_warning') diff --git a/app/views/admin/export_domain_blocks/import.html.haml b/app/views/admin/export_domain_blocks/import.html.haml new file mode 100644 index 000000000..01add232d --- /dev/null +++ b/app/views/admin/export_domain_blocks/import.html.haml @@ -0,0 +1,21 @@ +- content_for :page_title do + = t('admin.export_domain_blocks.import.title') + +%p= t('admin.export_domain_blocks.import.description_html') + +- if defined?(@global_private_comment) && @global_private_comment.present? + %p= t('admin.export_domain_blocks.import.private_comment_description_html', comment: @global_private_comment) + += form_for(@form, url: batch_admin_domain_blocks_path) do |f| + .batch-table + .batch-table__toolbar + %label.batch-table__toolbar__select.batch-checkbox-all + = check_box_tag :batch_checkbox_all, nil, false + .batch-table__toolbar__actions + = f.button safe_join([fa_icon('copy'), t('admin.domain_blocks.import')]), name: :save, class: 'table-action-link', type: :submit, data: { confirm: t('admin.reports.are_you_sure') } + .batch-table__body + - if @domain_blocks.empty? + = nothing_here 'nothing-here--under-tabs' + - else + = f.simple_fields_for :domain_blocks, @domain_blocks do |ff| + = render 'domain_block', f: ff, existing_relationships: @warning_domains.include?(ff.object.domain) diff --git a/app/views/admin/export_domain_blocks/new.html.haml b/app/views/admin/export_domain_blocks/new.html.haml new file mode 100644 index 000000000..0291aeed7 --- /dev/null +++ b/app/views/admin/export_domain_blocks/new.html.haml @@ -0,0 +1,10 @@ +- content_for :page_title do + = t('.title') + += simple_form_for @import, url: import_admin_export_domain_blocks_path, html: { multipart: true } do |f| + .fields-row + .fields-group.fields-row__column.fields-row__column-6 + = f.input :data, wrapper: :with_block_label, hint: t('simple_form.hints.imports.data'), as: :file + + .actions + = f.button :button, t('imports.upload'), type: :submit diff --git a/app/views/admin/instances/index.html.haml b/app/views/admin/instances/index.html.haml index cc5020398..8f7e3e67d 100644 --- a/app/views/admin/instances/index.html.haml +++ b/app/views/admin/instances/index.html.haml @@ -7,8 +7,12 @@ - content_for :heading_actions do - if whitelist_mode? = link_to t('admin.domain_allows.add_new'), new_admin_domain_allow_path, class: 'button', id: 'add-instance-button' + = link_to t('admin.domain_allows.export'), export_admin_export_domain_allows_path(format: :csv), class: 'button' + = link_to t('admin.domain_allows.import'), new_admin_export_domain_allow_path, class: 'button' - else = link_to t('admin.domain_blocks.add_new'), new_admin_domain_block_path, class: 'button', id: 'add-instance-button' + = link_to t('admin.domain_blocks.export'), export_admin_export_domain_blocks_path(format: :csv), class: 'button' + = link_to t('admin.domain_blocks.import'), new_admin_export_domain_block_path, class: 'button' .filters .filter-subset diff --git a/config/locales/en.yml b/config/locales/en.yml index 3a80c125d..1cc53dca4 100644 --- a/config/locales/en.yml +++ b/config/locales/en.yml @@ -373,6 +373,8 @@ en: add_new: Allow federation with domain created_msg: Domain has been successfully allowed for federation destroyed_msg: Domain has been disallowed from federation + export: Export + import: Import undo: Disallow federation with domain domain_blocks: add_new: Add new domain block @@ -382,6 +384,8 @@ en: edit: Edit domain block existing_domain_block: You have already imposed stricter limits on %{name}. existing_domain_block_html: You have already imposed stricter limits on %{name}, you need to unblock it first. + export: Export + import: Import new: create: Create block hint: The domain block will not prevent creation of account entries in the database, but will retroactively and automatically apply specific moderation methods on those accounts. @@ -391,6 +395,8 @@ en: silence: Limit suspend: Suspend title: New domain block + no_domain_block_selected: No domain blocks were changed as none were selected + not_permitted: You are not permitted to perform this action obfuscate: Obfuscate domain name obfuscate_hint: Partially obfuscate the domain name in the list if advertising the list of domain limitations is enabled private_comment: Private comment @@ -422,6 +428,20 @@ en: resolved_dns_records_hint_html: The domain name resolves to the following MX domains, which are ultimately responsible for accepting e-mail. Blocking an MX domain will block sign-ups from any e-mail address which uses the same MX domain, even if the visible domain name is different. Be careful not to block major e-mail providers. resolved_through_html: Resolved through %{domain} title: Blocked e-mail domains + export_domain_allows: + new: + title: Import domain allows + no_file: No file selected + export_domain_blocks: + import: + description_html: You are about to import a list of domain blocks. Please review this list very carefully, especially if you have not authored this list yourself. + existing_relationships_warning: Existing follow relationships + private_comment_description_html: 'To help you track where imported blocks come from, imported blocks will be created with the following private comment: %{comment}' + private_comment_template: Imported from %{source} on %{date} + title: Import domain blocks + new: + title: Import domain blocks + no_file: No file selected follow_recommendations: description_html: "Follow recommendations help new users quickly find interesting content. When a user has not interacted with others enough to form personalized follow recommendations, these accounts are recommended instead. They are re-calculated on a daily basis from a mix of accounts with the highest recent engagements and highest local follower counts for a given language." language: For language @@ -1159,6 +1179,7 @@ en: invalid_markup: 'contains invalid HTML markup: %{error}' imports: errors: + invalid_csv_file: 'Invalid CSV file. Error: %{error}' over_rows_processing_limit: contains more than %{count} rows modes: merge: Merge diff --git a/config/routes.rb b/config/routes.rb index 98aa5a033..98e19667c 100644 --- a/config/routes.rb +++ b/config/routes.rb @@ -225,7 +225,25 @@ Rails.application.routes.draw do get '/dashboard', to: 'dashboard#index' resources :domain_allows, only: [:new, :create, :show, :destroy] - resources :domain_blocks, only: [:new, :create, :destroy, :update, :edit] + resources :domain_blocks, only: [:new, :create, :show, :destroy, :update, :edit] do + collection do + post :batch + end + end + + resources :export_domain_allows, only: [:new] do + collection do + get :export, constraints: { format: :csv } + post :import + end + end + + resources :export_domain_blocks, only: [:new] do + collection do + get :export, constraints: { format: :csv } + post :import + end + end resources :email_domain_blocks, only: [:index, :new, :create] do collection do @@ -523,6 +541,7 @@ Rails.application.routes.draw do end resource :domain_blocks, only: [:show, :create, :destroy] + resource :directory, only: [:show] resources :follow_requests, only: [:index] do diff --git a/spec/controllers/admin/domain_allows_controller_spec.rb b/spec/controllers/admin/domain_allows_controller_spec.rb new file mode 100644 index 000000000..6c4e67787 --- /dev/null +++ b/spec/controllers/admin/domain_allows_controller_spec.rb @@ -0,0 +1,48 @@ +require 'rails_helper' + +RSpec.describe Admin::DomainAllowsController, type: :controller do + render_views + + before do + sign_in Fabricate(:user, role: UserRole.find_by(name: 'Admin')), scope: :user + end + + describe 'GET #new' do + it 'assigns a new domain allow' do + get :new + + expect(assigns(:domain_allow)).to be_instance_of(DomainAllow) + expect(response).to have_http_status(200) + end + end + + describe 'POST #create' do + it 'blocks the domain when succeeded to save' do + post :create, params: { domain_allow: { domain: 'example.com' } } + + expect(flash[:notice]).to eq I18n.t('admin.domain_allows.created_msg') + expect(response).to redirect_to(admin_instances_path) + end + + it 'renders new when failed to save' do + Fabricate(:domain_allow, domain: 'example.com') + + post :create, params: { domain_allow: { domain: 'example.com' } } + + expect(response).to render_template :new + end + end + + describe 'DELETE #destroy' do + it 'disallows the domain' do + service = double(call: true) + allow(UnallowDomainService).to receive(:new).and_return(service) + domain_allow = Fabricate(:domain_allow) + delete :destroy, params: { id: domain_allow.id } + + expect(service).to have_received(:call).with(domain_allow) + expect(flash[:notice]).to eq I18n.t('admin.domain_allows.destroyed_msg') + expect(response).to redirect_to(admin_instances_path) + end + end +end diff --git a/spec/controllers/admin/domain_blocks_controller_spec.rb b/spec/controllers/admin/domain_blocks_controller_spec.rb index 5c2dcd268..98cda5004 100644 --- a/spec/controllers/admin/domain_blocks_controller_spec.rb +++ b/spec/controllers/admin/domain_blocks_controller_spec.rb @@ -16,6 +16,27 @@ RSpec.describe Admin::DomainBlocksController, type: :controller do end end + describe 'POST #batch' do + it 'blocks the domains when succeeded to save' do + allow(DomainBlockWorker).to receive(:perform_async).and_return(true) + + post :batch, params: { + save: '', + form_domain_block_batch: { + domain_blocks_attributes: { + '0' => { enabled: '1', domain: 'example.com', severity: 'silence' }, + '1' => { enabled: '0', domain: 'mastodon.social', severity: 'suspend' }, + '2' => { enabled: '1', domain: 'mastodon.online', severity: 'suspend' } + } + } + } + + expect(DomainBlockWorker).to have_received(:perform_async).exactly(2).times + expect(flash[:notice]).to eq I18n.t('admin.domain_blocks.created_msg') + expect(response).to redirect_to(admin_instances_path(limited: '1')) + end + end + describe 'POST #create' do it 'blocks the domain when succeeded to save' do allow(DomainBlockWorker).to receive(:perform_async).and_return(true) diff --git a/spec/controllers/admin/export_domain_allows_controller_spec.rb b/spec/controllers/admin/export_domain_allows_controller_spec.rb new file mode 100644 index 000000000..1e1a5ae7d --- /dev/null +++ b/spec/controllers/admin/export_domain_allows_controller_spec.rb @@ -0,0 +1,42 @@ +require 'rails_helper' + +RSpec.describe Admin::ExportDomainAllowsController, type: :controller do + render_views + + before do + sign_in Fabricate(:user, role: UserRole.find_by(name: 'Admin')), scope: :user + end + + describe 'GET #export' do + it 'renders instances' do + Fabricate(:domain_allow, domain: 'good.domain') + Fabricate(:domain_allow, domain: 'better.domain') + + get :export, params: { format: :csv } + expect(response).to have_http_status(200) + expect(response.body).to eq(IO.read(File.join(file_fixture_path, 'domain_allows.csv'))) + end + end + + describe 'POST #import' do + it 'allows imported domains' do + post :import, params: { admin_import: { data: fixture_file_upload('domain_allows.csv') } } + + expect(response).to redirect_to(admin_instances_path) + + # Header should not be imported + expect(DomainAllow.where(domain: '#domain').present?).to eq(false) + + # Domains should now be added + get :export, params: { format: :csv } + expect(response).to have_http_status(200) + expect(response.body).to eq(IO.read(File.join(file_fixture_path, 'domain_allows.csv'))) + end + + it 'displays error on no file selected' do + post :import, params: { admin_import: {} } + expect(response).to redirect_to(admin_instances_path) + expect(flash[:error]).to eq(I18n.t('admin.export_domain_allows.no_file')) + end + end +end diff --git a/spec/controllers/admin/export_domain_blocks_controller_spec.rb b/spec/controllers/admin/export_domain_blocks_controller_spec.rb new file mode 100644 index 000000000..8697e0c21 --- /dev/null +++ b/spec/controllers/admin/export_domain_blocks_controller_spec.rb @@ -0,0 +1,35 @@ +require 'rails_helper' + +RSpec.describe Admin::ExportDomainBlocksController, type: :controller do + render_views + + before do + sign_in Fabricate(:user, role: UserRole.find_by(name: 'Admin')), scope: :user + end + + describe 'GET #export' do + it 'renders instances' do + Fabricate(:domain_block, domain: 'bad.domain', severity: 'silence', public_comment: 'bad') + Fabricate(:domain_block, domain: 'worse.domain', severity: 'suspend', reject_media: true, reject_reports: true, public_comment: 'worse', obfuscate: true) + Fabricate(:domain_block, domain: 'reject.media', severity: 'noop', reject_media: true, public_comment: 'reject media') + Fabricate(:domain_block, domain: 'no.op', severity: 'noop', public_comment: 'noop') + + get :export, params: { format: :csv } + expect(response).to have_http_status(200) + expect(response.body).to eq(IO.read(File.join(file_fixture_path, 'domain_blocks.csv'))) + end + end + + describe 'POST #import' do + it 'blocks imported domains' do + post :import, params: { admin_import: { data: fixture_file_upload('domain_blocks.csv') } } + + expect(assigns(:domain_blocks).map(&:domain)).to match_array ['bad.domain', 'worse.domain', 'reject.media'] + end + end + + it 'displays error on no file selected' do + post :import, params: { admin_import: {} } + expect(flash[:alert]).to eq(I18n.t('admin.export_domain_blocks.no_file')) + end +end diff --git a/spec/fixtures/files/domain_allows.csv b/spec/fixtures/files/domain_allows.csv new file mode 100644 index 000000000..4200ac3f5 --- /dev/null +++ b/spec/fixtures/files/domain_allows.csv @@ -0,0 +1,3 @@ +#domain +good.domain +better.domain diff --git a/spec/fixtures/files/domain_blocks.csv b/spec/fixtures/files/domain_blocks.csv new file mode 100644 index 000000000..28ffb9175 --- /dev/null +++ b/spec/fixtures/files/domain_blocks.csv @@ -0,0 +1,4 @@ +#domain,#severity,#reject_media,#reject_reports,#public_comment,#obfuscate +bad.domain,silence,false,false,bad,false +worse.domain,suspend,true,true,worse,true +reject.media,noop,true,false,reject media,false -- cgit From ab7d99e035f5b880ef77440e7c2e76f8e8728992 Mon Sep 17 00:00:00 2001 From: Claire Date: Thu, 17 Nov 2022 11:42:20 +0100 Subject: Fix various issues with domain block import (#1944) - stop using Paperclip for processing domain allow/block imports - stop leaving temporary files - better error handling - assume CSV files are UTF-8-encoded --- .../admin/export_domain_allows_controller.rb | 6 ++-- .../admin/export_domain_blocks_controller.rb | 6 ++-- .../concerns/admin_export_controller_concern.rb | 8 ++--- app/models/admin/import.rb | 35 ++++++++++++---------- app/validators/admin_import_validator.rb | 19 ------------ config/locales/en.yml | 1 + 6 files changed, 30 insertions(+), 45 deletions(-) delete mode 100644 app/validators/admin_import_validator.rb (limited to 'app/models') diff --git a/app/controllers/admin/export_domain_allows_controller.rb b/app/controllers/admin/export_domain_allows_controller.rb index eb2955ac3..57fb12c62 100644 --- a/app/controllers/admin/export_domain_allows_controller.rb +++ b/app/controllers/admin/export_domain_allows_controller.rb @@ -8,8 +8,6 @@ module Admin before_action :set_dummy_import!, only: [:new] - ROWS_PROCESSING_LIMIT = 20_000 - def new authorize :domain_allow, :create? end @@ -23,9 +21,11 @@ module Admin authorize :domain_allow, :create? begin @import = Admin::Import.new(import_params) + return render :new unless @import.validate + parse_import_data!(export_headers) - @data.take(ROWS_PROCESSING_LIMIT).each do |row| + @data.take(Admin::Import::ROWS_PROCESSING_LIMIT).each do |row| domain = row['#domain'].strip next if DomainAllow.allowed?(domain) diff --git a/app/controllers/admin/export_domain_blocks_controller.rb b/app/controllers/admin/export_domain_blocks_controller.rb index 545bd94ed..fb0cd05d2 100644 --- a/app/controllers/admin/export_domain_blocks_controller.rb +++ b/app/controllers/admin/export_domain_blocks_controller.rb @@ -8,8 +8,6 @@ module Admin before_action :set_dummy_import!, only: [:new] - ROWS_PROCESSING_LIMIT = 20_000 - def new authorize :domain_block, :create? end @@ -23,12 +21,14 @@ module Admin authorize :domain_block, :create? @import = Admin::Import.new(import_params) + return render :new unless @import.validate + parse_import_data!(export_headers) @global_private_comment = I18n.t('admin.export_domain_blocks.import.private_comment_template', source: @import.data_file_name, date: I18n.l(Time.now.utc)) @form = Form::DomainBlockBatch.new - @domain_blocks = @data.take(ROWS_PROCESSING_LIMIT).filter_map do |row| + @domain_blocks = @data.take(Admin::Import::ROWS_PROCESSING_LIMIT).filter_map do |row| domain = row['#domain'].strip next if DomainBlock.rule_for(domain).present? diff --git a/app/controllers/concerns/admin_export_controller_concern.rb b/app/controllers/concerns/admin_export_controller_concern.rb index 013915d02..b40c76557 100644 --- a/app/controllers/concerns/admin_export_controller_concern.rb +++ b/app/controllers/concerns/admin_export_controller_concern.rb @@ -27,13 +27,13 @@ module AdminExportControllerConcern params.require(:admin_import).permit(:data) end - def import_data - Paperclip.io_adapters.for(@import.data).read + def import_data_path + params[:admin_import][:data].path end def parse_import_data!(default_headers) - data = CSV.parse(import_data, headers: true) - data = CSV.parse(import_data, headers: default_headers) unless data.headers&.first&.strip&.include?(default_headers[0]) + data = CSV.read(import_data_path, headers: true, encoding: 'UTF-8') + data = CSV.read(import_data_path, headers: default_headers, encoding: 'UTF-8') unless data.headers&.first&.strip&.include?(default_headers[0]) @data = data.reject(&:blank?) end end diff --git a/app/models/admin/import.rb b/app/models/admin/import.rb index c305be237..79c0722d5 100644 --- a/app/models/admin/import.rb +++ b/app/models/admin/import.rb @@ -2,28 +2,31 @@ # A non-activerecord helper class for csv upload class Admin::Import - extend ActiveModel::Callbacks include ActiveModel::Model - include Paperclip::Glue - FILE_TYPES = %w(text/plain text/csv application/csv).freeze + ROWS_PROCESSING_LIMIT = 20_000 - # Paperclip required callbacks - define_model_callbacks :save, only: [:after] - define_model_callbacks :destroy, only: [:before, :after] + attr_accessor :data - attr_accessor :data_file_name, :data_content_type + validates :data, presence: true + validate :validate_data - has_attached_file :data - validates_attachment_content_type :data, content_type: FILE_TYPES - validates_attachment_presence :data - validates_with AdminImportValidator, on: :create - - def save - run_callbacks :save + def data_file_name + data.original_filename end - def destroy - run_callbacks :destroy + private + + def validate_data + return if data.blank? + + csv_data = CSV.read(data.path, encoding: 'UTF-8') + + row_count = csv_data.size + row_count -= 1 if csv_data.first&.first == '#domain' + + errors.add(:data, I18n.t('imports.errors.over_rows_processing_limit', count: ROWS_PROCESSING_LIMIT)) if row_count > ROWS_PROCESSING_LIMIT + rescue CSV::MalformedCSVError => e + errors.add(:data, I18n.t('imports.errors.invalid_csv_file', error: e.message)) end end diff --git a/app/validators/admin_import_validator.rb b/app/validators/admin_import_validator.rb deleted file mode 100644 index 338ceb3a7..000000000 --- a/app/validators/admin_import_validator.rb +++ /dev/null @@ -1,19 +0,0 @@ -# frozen_string_literal: true - -class AdminImportValidator < ActiveModel::Validator - FIRST_HEADER = '#domain' - - def validate(import) - return if import.type.blank? || import.data.blank? - - # We parse because newlines could be part of individual rows. This - # runs on create so we should be reading the local file here before - # it is uploaded to object storage or moved anywhere... - csv_data = CSV.parse(import.data.queued_for_write[:original].read) - - row_count = csv_data.size - row_count -= 1 if csv_data.first&.first == FIRST_HEADER - - import.errors.add(:data, I18n.t('imports.errors.over_rows_processing_limit', count: Admin::DomainBlocksController::ROWS_PROCESSING_LIMIT)) if row_count > Admin::DomainBlocksController::ROWS_PROCESSING_LIMIT - end -end diff --git a/config/locales/en.yml b/config/locales/en.yml index 679e356b4..2f766cef3 100644 --- a/config/locales/en.yml +++ b/config/locales/en.yml @@ -1159,6 +1159,7 @@ en: invalid_markup: 'contains invalid HTML markup: %{error}' imports: errors: + invalid_csv_file: 'Invalid CSV file. Error: %{error}' over_rows_processing_limit: contains more than %{count} rows modes: merge: Merge -- cgit From 69378eac99c013a0db7d2d5ff9a54dfcc287d9ce Mon Sep 17 00:00:00 2001 From: David Leadbeater Date: Mon, 21 Nov 2022 05:28:13 +1100 Subject: Don't allow URLs that contain non-normalized paths to be verified (#20999) * Don't allow URLs that contain non-normalized paths to be verified This stops things like https://example.com/otheruser/../realuser where "/otheruser" appears to be the verified URL, but the actual URL being verified is "/realuser" due to the "/../". Also fix a test to use 'https', so it is testing the right thing, now that since #20304 https is required. * missing do --- app/models/account/field.rb | 3 ++- spec/models/account/field_spec.rb | 10 +++++++++- 2 files changed, 11 insertions(+), 2 deletions(-) (limited to 'app/models') diff --git a/app/models/account/field.rb b/app/models/account/field.rb index ffc8dce80..4db4cac30 100644 --- a/app/models/account/field.rb +++ b/app/models/account/field.rb @@ -46,7 +46,8 @@ class Account::Field < ActiveModelSerializers::Model parsed_url.user.nil? && parsed_url.password.nil? && parsed_url.host.present? && - parsed_url.normalized_host == parsed_url.host + parsed_url.normalized_host == parsed_url.host && + (parsed_url.path.empty? || parsed_url.path == parsed_url.normalized_path) rescue Addressable::URI::InvalidURIError, IDN::Idna::IdnaError false end diff --git a/spec/models/account/field_spec.rb b/spec/models/account/field_spec.rb index b4beec048..0ac9769bc 100644 --- a/spec/models/account/field_spec.rb +++ b/spec/models/account/field_spec.rb @@ -67,7 +67,15 @@ RSpec.describe Account::Field, type: :model do end context 'for an IDN URL' do - let(:value) { 'http://twitter.com∕dougallj∕status∕1590357240443437057.ê.cc/twitter.html' } + let(:value) { 'https://twitter.com∕dougallj∕status∕1590357240443437057.ê.cc/twitter.html' } + + it 'returns false' do + expect(subject.verifiable?).to be false + end + end + + context 'for a URL with a non-normalized path' do + let(:value) { 'https://github.com/octocatxxxxxxxx/../mastodon' } it 'returns false' do expect(subject.verifiable?).to be false -- cgit From 5b2ff8d32d66a3820799660ad81275d1b6df24fe Mon Sep 17 00:00:00 2001 From: Skyler Hawthorne Date: Fri, 25 Nov 2022 10:20:47 -0500 Subject: fix media uploads with ffmpeg 5 (#21191) --- app/models/media_attachment.rb | 1 + 1 file changed, 1 insertion(+) (limited to 'app/models') diff --git a/app/models/media_attachment.rb b/app/models/media_attachment.rb index d2bdc55f0..51b256482 100644 --- a/app/models/media_attachment.rb +++ b/app/models/media_attachment.rb @@ -104,6 +104,7 @@ class MediaAttachment < ApplicationRecord 'c:v' => 'h264', 'maxrate' => '1300K', 'bufsize' => '1300K', + 'b:v' => '1300K', 'frames:v' => 60 * 60 * 3, 'crf' => 18, 'map_metadata' => '-1', -- cgit From f17fc5742e539db10e0e430f5218c440849b894c Mon Sep 17 00:00:00 2001 From: afontenot Date: Sat, 26 Nov 2022 14:08:25 -0800 Subject: Clear voter count when poll is reset (#21700) When a poll is edited, we reset the poll and remove all previous votes. However, prior to this commit, the voter count on the poll was not reset. This leads to incorrect percentages being shown in poll results. Fixes #21696 --- app/models/poll.rb | 1 + 1 file changed, 1 insertion(+) (limited to 'app/models') diff --git a/app/models/poll.rb b/app/models/poll.rb index 1a326e452..af3b09315 100644 --- a/app/models/poll.rb +++ b/app/models/poll.rb @@ -85,6 +85,7 @@ class Poll < ApplicationRecord def reset_votes! self.cached_tallies = options.map { 0 } self.votes_count = 0 + self.voters_count = 0 votes.delete_all unless new_record? end -- cgit From 47f0d7021e76c60f5336b0c005c6336cd50c8009 Mon Sep 17 00:00:00 2001 From: Kaspar V Date: Sun, 27 Nov 2022 20:41:18 +0100 Subject: refactor(vacuum statuses): reduce amount of db queries and load for each query - improve performance (#21487) * refactor(statuses_vacuum): remove dead code - unused Method is not called inside class and private. Clean up dead code. * refactor(statuses_vacuum): make retention_period present test explicit This private method only hides functionality. It is best practice to be as explicit as possible. * refactor(statuses_vacuum): improve query performance - fix statuses_scope having sub-select for Account.remote scope by `joins(:account).merge(Account.remote)` - fix statuses_scope unnecessary use of `Status.arel_table[:id].lt` because it is inexplicit, bad practice and even slower than normal `.where('statuses.id < ?'` - fix statuses_scope remove select(:id, :visibility) for having reusable active record query batches (no re queries) - fix vacuum_statuses! to use in_batches instead of find_in_batches, because in_batches delivers a full blown active record query result, in stead of an array - no requeries necessary - send(:unlink_from_conversations) not to perform another db query, but reuse the in_batches result instead. - remove now obsolete remove_from_account_conversations method - remove_from_search_index uses array of ids, instead of mapping the ids from an array - this should be more efficient - use the in_batches scope to call delete_all, instead of running another db query for this - because it is again more efficient - add TODO comment for calling models private method with send * refactor(status): simplify unlink_from_conversations - add `has_many through:` relation mentioned_accounts - use model scope local instead of method call `Status#local?` - more readable add account to inbox_owners when account.local? * refactor(status): searchable_by way less sub selects These queries all included a sub-select. Doing the same with a joins should be more efficient. Since this method does 5 such queries, this should be significant, since it technically halves the query count. This is how it was: ```ruby [3] pry(main)> Status.first.mentions.where(account: Account.local, silent: false).explain Status Load (1.6ms) SELECT "statuses".* FROM "statuses" WHERE "statuses"."deleted_at" IS NULL ORDER BY "statuses"."id" DESC LIMIT $1 [["LIMIT", 1]] Mention Load (1.5ms) SELECT "mentions".* FROM "mentions" WHERE "mentions"."status_id" = $1 AND "mentions"."account_id" IN (SELECT "accounts"."id" FROM "accounts" WHERE "accounts"."domain" IS NULL) AND "mentions"."silent" = $2 [["status_id", 109382923142288414], ["silent", false]] => EXPLAIN for: SELECT "mentions".* FROM "mentions" WHERE "mentions"."status_id" = $1 AND "mentions"."account_id" IN (SELECT "accounts"."id" FROM "accounts" WHERE "accounts"."domain" IS NULL) AND "mentions"."silent" = $2 [["status_id", 109382923142288414], ["silent", false]] QUERY PLAN ------------------------------------------------------------------------------------------------------------------ Nested Loop (cost=0.15..23.08 rows=1 width=41) -> Seq Scan on accounts (cost=0.00..10.90 rows=1 width=8) Filter: (domain IS NULL) -> Index Scan using index_mentions_on_account_id_and_status_id on mentions (cost=0.15..8.17 rows=1 width=41) Index Cond: ((account_id = accounts.id) AND (status_id = '109382923142288414'::bigint)) Filter: (NOT silent) (6 rows) ``` This is how it is with this change: ```ruby [4] pry(main)> Status.first.mentions.joins(:account).merge(Account.local).active.explain Status Load (1.7ms) SELECT "statuses".* FROM "statuses" WHERE "statuses"."deleted_at" IS NULL ORDER BY "statuses"."id" DESC LIMIT $1 [["LIMIT", 1]] Mention Load (0.7ms) SELECT "mentions".* FROM "mentions" INNER JOIN "accounts" ON "accounts"."id" = "mentions"."account_id" WHERE "mentions"."status_id" = $1 AND "accounts"."domain" IS NULL AND "mentions"."silent" = $2 [["status_id", 109382923142288414], ["silent", false]] => EXPLAIN for: SELECT "mentions".* FROM "mentions" INNER JOIN "accounts" ON "accounts"."id" = "mentions"."account_id" WHERE "mentions"."status_id" = $1 AND "accounts"."domain" IS NULL AND "mentions"."silent" = $2 [["status_id", 109382923142288414], ["silent", false]] QUERY PLAN ------------------------------------------------------------------------------------------------------------------ Nested Loop (cost=0.15..23.08 rows=1 width=41) -> Seq Scan on accounts (cost=0.00..10.90 rows=1 width=8) Filter: (domain IS NULL) -> Index Scan using index_mentions_on_account_id_and_status_id on mentions (cost=0.15..8.17 rows=1 width=41) Index Cond: ((account_id = accounts.id) AND (status_id = '109382923142288414'::bigint)) Filter: (NOT silent) (6 rows) ``` --- app/lib/vacuum/statuses_vacuum.rb | 43 ++++++++++++++++----------------------- app/models/status.rb | 15 +++++++------- 2 files changed, 26 insertions(+), 32 deletions(-) (limited to 'app/models') diff --git a/app/lib/vacuum/statuses_vacuum.rb b/app/lib/vacuum/statuses_vacuum.rb index 41d6ba270..d1c4e7197 100644 --- a/app/lib/vacuum/statuses_vacuum.rb +++ b/app/lib/vacuum/statuses_vacuum.rb @@ -8,47 +8,40 @@ class Vacuum::StatusesVacuum end def perform - vacuum_statuses! if retention_period? + vacuum_statuses! if @retention_period.present? end private def vacuum_statuses! - statuses_scope.find_in_batches do |statuses| + statuses_scope.in_batches do |statuses| # Side-effects not covered by foreign keys, such # as the search index, must be handled first. - - remove_from_account_conversations(statuses) - remove_from_search_index(statuses) - - # Foreign keys take care of most associated records - # for us. Media attachments will be orphaned. - - Status.where(id: statuses.map(&:id)).delete_all + statuses.direct_visibility + .includes(mentions: :account) + .find_each do |status| + # TODO: replace temporary solution - call of private model method + status.send(:unlink_from_conversations) + end + remove_from_search_index(statuses.ids) if Chewy.enabled? + + # Foreign keys take care of most associated records for us. + # Media attachments will be orphaned. + statuses.delete_all end end def statuses_scope - Status.unscoped.kept.where(account: Account.remote).where(Status.arel_table[:id].lt(retention_period_as_id)).select(:id, :visibility) + Status.unscoped.kept + .joins(:account).merge(Account.remote) + .where('statuses.id < ?', retention_period_as_id) end def retention_period_as_id Mastodon::Snowflake.id_at(@retention_period.ago, with_random: false) end - def analyze_statuses! - ActiveRecord::Base.connection.execute('ANALYZE statuses') - end - - def remove_from_account_conversations(statuses) - Status.where(id: statuses.select(&:direct_visibility?).map(&:id)).includes(:account, mentions: :account).each(&:unlink_from_conversations) - end - - def remove_from_search_index(statuses) - with_redis { |redis| redis.sadd('chewy:queue:StatusesIndex', statuses.map(&:id)) } if Chewy.enabled? - end - - def retention_period? - @retention_period.present? + def remove_from_search_index(status_ids) + with_redis { |redis| redis.sadd('chewy:queue:StatusesIndex', status_ids) } end end diff --git a/app/models/status.rb b/app/models/status.rb index 8bdb5e8db..2fe9f2de0 100644 --- a/app/models/status.rb +++ b/app/models/status.rb @@ -66,6 +66,7 @@ class Status < ApplicationRecord has_many :reblogged_by_accounts, through: :reblogs, class_name: 'Account', source: :account has_many :replies, foreign_key: 'in_reply_to_id', class_name: 'Status', inverse_of: :thread has_many :mentions, dependent: :destroy, inverse_of: :status + has_many :mentioned_accounts, through: :mentions, source: :account, class_name: 'Account' has_many :active_mentions, -> { active }, class_name: 'Mention', inverse_of: :status has_many :media_attachments, dependent: :nullify @@ -141,11 +142,11 @@ class Status < ApplicationRecord ids << account_id if local? if preloaded.nil? - ids += mentions.where(account: Account.local, silent: false).pluck(:account_id) - ids += favourites.where(account: Account.local).pluck(:account_id) - ids += reblogs.where(account: Account.local).pluck(:account_id) - ids += bookmarks.where(account: Account.local).pluck(:account_id) - ids += poll.votes.where(account: Account.local).pluck(:account_id) if poll.present? + ids += mentions.joins(:account).merge(Account.local).active.pluck(:account_id) + ids += favourites.joins(:account).merge(Account.local).pluck(:account_id) + ids += reblogs.joins(:account).merge(Account.local).pluck(:account_id) + ids += bookmarks.joins(:account).merge(Account.local).pluck(:account_id) + ids += poll.votes.joins(:account).merge(Account.local).pluck(:account_id) if poll.present? else ids += preloaded.mentions[id] || [] ids += preloaded.favourites[id] || [] @@ -527,8 +528,8 @@ class Status < ApplicationRecord def unlink_from_conversations return unless direct_visibility? - mentioned_accounts = (association(:mentions).loaded? ? mentions : mentions.includes(:account)).map(&:account) - inbox_owners = mentioned_accounts.select(&:local?) + (account.local? ? [account] : []) + inbox_owners = mentioned_accounts.local + inbox_owners += [account] if account.local? inbox_owners.each do |inbox_owner| AccountConversation.remove_status(inbox_owner, self) -- cgit From 57b893d505a6fd0a11b9e53261bc029adee2aa24 Mon Sep 17 00:00:00 2001 From: Claire Date: Sun, 27 Nov 2022 20:47:29 +0100 Subject: Fix spaces not being stripped in admin account search (#21324) Fixes #21058 Regression from #18641 --- app/models/account_filter.rb | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) (limited to 'app/models') diff --git a/app/models/account_filter.rb b/app/models/account_filter.rb index e09ce4ec2..3a4ac0492 100644 --- a/app/models/account_filter.rb +++ b/app/models/account_filter.rb @@ -43,13 +43,13 @@ class AccountFilter when 'status' status_scope(value) when 'by_domain' - Account.where(domain: value.to_s) + Account.where(domain: value.to_s.strip) when 'username' - Account.matches_username(value.to_s) + Account.matches_username(value.to_s.strip) when 'display_name' - Account.matches_display_name(value.to_s) + Account.matches_display_name(value.to_s.strip) when 'email' - accounts_with_users.merge(User.matches_email(value.to_s)) + accounts_with_users.merge(User.matches_email(value.to_s.strip)) when 'ip' valid_ip?(value) ? accounts_with_users.merge(User.matches_ip(value).group('users.id, accounts.id')) : Account.none when 'invited_by' -- cgit