Support for import/export of instance-level domain blocks/allows for 4.x w/ additional fixes (#20597)

* Allow import/export of instance-level domain blocks/allows (#1754)

* Allow import/export of instance-level domain blocks/allows.
Fixes #15095

* Pacify circleci

* Address simple code review feedback

* Add headers to exported CSV

* Extract common import/export functionality to
AdminExportControllerConcern

* Add additional fields to instance-blocked domain export

* Address review feedback

* Split instance domain block/allow import/export into separate pages/controllers

* Address code review feedback

* Pacify DeepSource

* Work around Paperclip::HasAttachmentFile for Rails 6

* Fix deprecated API warning in export tests

* Remove after_commit workaround

(cherry picked from commit 94e98864e39c010635e839fea984f2b4893bef1a)

* Add confirmation page when importing blocked domains (#1773)

* Move glitch-soc-specific strings to glitch-soc-specific locale files

* Add confirmation page when importing blocked domains

(cherry picked from commit b91196f4b73fff91997b8077619ae25b6d04a59e)

* Fix authorization check in domain blocks controller

(cherry picked from commit 75279377583c6e2aa04cc8d7380c593979630b38)

* Fix error strings for domain blocks and email-domain blocks

Corrected issue with non-error message used for Mastodon:NotPermittedError in Domain Blocks
Corrected issue Domain Blocks using the Email Domain Blocks message on ActionContoller::ParameterMissing
Corrected issue with Email Domain Blocks using the not_permitted string from "custom emojii's"

* Ran i18n-tasks normalize to address test failure

* Removed unused admin.export_domain_blocks.not_permitted string

Removing unused string as indicated by Check i18n

* Fix tests

(cherry picked from commit 9094c2f52c24e1c00b594e7c11cd00e4a07eb431)

* Fix domain block export not exporting blocks with only media rejection

(cherry picked from commit 26ff48ee48a5c03a2a4b0bd03fd322529e6bd960)

* Fix various issues with domain block import

- stop using Paperclip for processing domain allow/block imports
- stop leaving temporary files
- better error handling
- assume CSV files are UTF-8-encoded

(cherry picked from commit cad824d8f501b95377e4f0a957e5a00d517a1902)

Co-authored-by: Levi Bard <taktaktaktaktaktaktaktaktaktak@gmail.com>
Co-authored-by: Claire <claire.github-309c@sitedethib.com>

15 files changed, 343 insertions, 1 deletions

diff --git a/app/controllers/admin/domain_blocks_controller.rb b/app/controllers/admin/domain_blocks_controller.rb
index 16defc1ea..e79f7a43e 100644
--- a/app/controllers/admin/domain_blocks_controller.rb
+++ b/app/controllers/admin/domain_blocks_controller.rb
@@ -4,6 +4,18 @@ module Admin
   class DomainBlocksController < BaseController
     before_action :set_domain_block, only: [:show, :destroy, :edit, :update]
 
+    def batch
+      authorize :domain_block, :create?
+      @form = Form::DomainBlockBatch.new(form_domain_block_batch_params.merge(current_account: current_account, action: action_from_button))
+      @form.save
+    rescue ActionController::ParameterMissing
+      flash[:alert] = I18n.t('admin.domain_blocks.no_domain_block_selected')
+    rescue Mastodon::NotPermittedError
+      flash[:alert] = I18n.t('admin.domain_blocks.not_permitted')
+    else
+      redirect_to admin_instances_path(limited: '1'), notice: I18n.t('admin.domain_blocks.created_msg')
+    end
+
     def new
       authorize :domain_block, :create?
       @domain_block = DomainBlock.new(domain: params[:_domain])
@@ -76,5 +88,15 @@ module Admin
     def resource_params
       params.require(:domain_block).permit(:domain, :severity, :reject_media, :reject_reports, :private_comment, :public_comment, :obfuscate)
     end
+
+    def form_domain_block_batch_params
+      params.require(:form_domain_block_batch).permit(domain_blocks_attributes: [:enabled, :domain, :severity, :reject_media, :reject_reports, :private_comment, :public_comment, :obfuscate])
+    end
+
+    def action_from_button
+      if params[:save]
+        'save'
+      end
+    end
   end
 end
diff --git a/app/controllers/admin/email_domain_blocks_controller.rb b/app/controllers/admin/email_domain_blocks_controller.rb
index 593457b94..a0a43de19 100644
--- a/app/controllers/admin/email_domain_blocks_controller.rb
+++ b/app/controllers/admin/email_domain_blocks_controller.rb
@@ -19,7 +19,7 @@ module Admin
     rescue ActionController::ParameterMissing
       flash[:alert] = I18n.t('admin.email_domain_blocks.no_email_domain_block_selected')
     rescue Mastodon::NotPermittedError
-      flash[:alert] = I18n.t('admin.custom_emojis.not_permitted')
+      flash[:alert] = I18n.t('admin.email_domain_blocks.not_permitted')
     ensure
       redirect_to admin_email_domain_blocks_path
     end
diff --git a/app/controllers/admin/export_domain_allows_controller.rb b/app/controllers/admin/export_domain_allows_controller.rb
new file mode 100644
index 000000000..57fb12c62
--- /dev/null
+++ b/app/controllers/admin/export_domain_allows_controller.rb
@@ -0,0 +1,60 @@
+# frozen_string_literal: true
+
+require 'csv'
+
+module Admin
+  class ExportDomainAllowsController < BaseController
+    include AdminExportControllerConcern
+
+    before_action :set_dummy_import!, only: [:new]
+
+    def new
+      authorize :domain_allow, :create?
+    end
+
+    def export
+      authorize :instance, :index?
+      send_export_file
+    end
+
+    def import
+      authorize :domain_allow, :create?
+      begin
+        @import = Admin::Import.new(import_params)
+        return render :new unless @import.validate
+
+        parse_import_data!(export_headers)
+
+        @data.take(Admin::Import::ROWS_PROCESSING_LIMIT).each do |row|
+          domain = row['#domain'].strip
+          next if DomainAllow.allowed?(domain)
+
+          domain_allow = DomainAllow.new(domain: domain)
+          log_action :create, domain_allow if domain_allow.save
+        end
+        flash[:notice] = I18n.t('admin.domain_allows.created_msg')
+      rescue ActionController::ParameterMissing
+        flash[:error] = I18n.t('admin.export_domain_allows.no_file')
+      end
+      redirect_to admin_instances_path
+    end
+
+    private
+
+    def export_filename
+      'domain_allows.csv'
+    end
+
+    def export_headers
+      %w(#domain)
+    end
+
+    def export_data
+      CSV.generate(headers: export_headers, write_headers: true) do |content|
+        DomainAllow.allowed_domains.each do |instance|
+          content << [instance.domain]
+        end
+      end
+    end
+  end
+end
diff --git a/app/controllers/admin/export_domain_blocks_controller.rb b/app/controllers/admin/export_domain_blocks_controller.rb
new file mode 100644
index 000000000..fb0cd05d2
--- /dev/null
+++ b/app/controllers/admin/export_domain_blocks_controller.rb
@@ -0,0 +1,71 @@
+# frozen_string_literal: true
+
+require 'csv'
+
+module Admin
+  class ExportDomainBlocksController < BaseController
+    include AdminExportControllerConcern
+
+    before_action :set_dummy_import!, only: [:new]
+
+    def new
+      authorize :domain_block, :create?
+    end
+
+    def export
+      authorize :instance, :index?
+      send_export_file
+    end
+
+    def import
+      authorize :domain_block, :create?
+
+      @import = Admin::Import.new(import_params)
+      return render :new unless @import.validate
+
+      parse_import_data!(export_headers)
+
+      @global_private_comment = I18n.t('admin.export_domain_blocks.import.private_comment_template', source: @import.data_file_name, date: I18n.l(Time.now.utc))
+
+      @form = Form::DomainBlockBatch.new
+      @domain_blocks = @data.take(Admin::Import::ROWS_PROCESSING_LIMIT).filter_map do |row|
+        domain = row['#domain'].strip
+        next if DomainBlock.rule_for(domain).present?
+
+        domain_block = DomainBlock.new(domain: domain,
+                                       severity: row['#severity'].strip,
+                                       reject_media: row['#reject_media'].strip,
+                                       reject_reports: row['#reject_reports'].strip,
+                                       private_comment: @global_private_comment,
+                                       public_comment: row['#public_comment']&.strip,
+                                       obfuscate: row['#obfuscate'].strip)
+
+        domain_block if domain_block.valid?
+      end
+
+      @warning_domains = Instance.where(domain: @domain_blocks.map(&:domain)).where('EXISTS (SELECT 1 FROM follows JOIN accounts ON follows.account_id = accounts.id OR follows.target_account_id = accounts.id WHERE accounts.domain = instances.domain)').pluck(:domain)
+    rescue ActionController::ParameterMissing
+      flash.now[:alert] = I18n.t('admin.export_domain_blocks.no_file')
+      set_dummy_import!
+      render :new
+    end
+
+    private
+
+    def export_filename
+      'domain_blocks.csv'
+    end
+
+    def export_headers
+      %w(#domain #severity #reject_media #reject_reports #public_comment #obfuscate)
+    end
+
+    def export_data
+      CSV.generate(headers: export_headers, write_headers: true) do |content|
+        DomainBlock.with_limitations.each do |instance|
+          content << [instance.domain, instance.severity, instance.reject_media, instance.reject_reports, instance.public_comment, instance.obfuscate]
+        end
+      end
+    end
+  end
+end
diff --git a/app/controllers/concerns/admin_export_controller_concern.rb b/app/controllers/concerns/admin_export_controller_concern.rb
new file mode 100644
index 000000000..b40c76557
--- /dev/null
+++ b/app/controllers/concerns/admin_export_controller_concern.rb
@@ -0,0 +1,39 @@
+# frozen_string_literal: true
+
+module AdminExportControllerConcern
+  extend ActiveSupport::Concern
+
+  private
+
+  def send_export_file
+    respond_to do |format|
+      format.csv { send_data export_data, filename: export_filename }
+    end
+  end
+
+  def export_data
+    raise 'Override in controller'
+  end
+
+  def export_filename
+    raise 'Override in controller'
+  end
+
+  def set_dummy_import!
+    @import = Admin::Import.new
+  end
+
+  def import_params
+    params.require(:admin_import).permit(:data)
+  end
+
+  def import_data_path
+    params[:admin_import][:data].path
+  end
+
+  def parse_import_data!(default_headers)
+    data = CSV.read(import_data_path, headers: true, encoding: 'UTF-8')
+    data = CSV.read(import_data_path, headers: default_headers, encoding: 'UTF-8') unless data.headers&.first&.strip&.include?(default_headers[0])
+    @data = data.reject(&:blank?)
+  end
+end
diff --git a/app/javascript/packs/admin.js b/app/javascript/packs/admin.js
index de86e0e11..4e817129d 100644
--- a/app/javascript/packs/admin.js
+++ b/app/javascript/packs/admin.js
@@ -185,6 +185,12 @@ ready(() => {
   const registrationMode = document.getElementById('form_admin_settings_registrations_mode');
   if (registrationMode) onChangeRegistrationMode(registrationMode);
 
+  const checkAllElement = document.querySelector('#batch_checkbox_all');
+  if (checkAllElement) {
+    checkAllElement.checked = [].every.call(document.querySelectorAll(batchCheckboxClassName), (content) => content.checked);
+    checkAllElement.indeterminate = !checkAllElement.checked && [].some.call(document.querySelectorAll(batchCheckboxClassName), (content) => content.checked);
+  }
+
   document.querySelector('a#add-instance-button')?.addEventListener('click', (e) => {
     const domain = document.getElementById('by_domain')?.value;
 
diff --git a/app/models/admin/import.rb b/app/models/admin/import.rb
new file mode 100644
index 000000000..79c0722d5
--- /dev/null
+++ b/app/models/admin/import.rb
@@ -0,0 +1,32 @@
+# frozen_string_literal: true
+
+# A non-activerecord helper class for csv upload
+class Admin::Import
+  include ActiveModel::Model
+
+  ROWS_PROCESSING_LIMIT = 20_000
+
+  attr_accessor :data
+
+  validates :data, presence: true
+  validate :validate_data
+
+  def data_file_name
+    data.original_filename
+  end
+
+  private
+
+  def validate_data
+    return if data.blank?
+
+    csv_data = CSV.read(data.path, encoding: 'UTF-8')
+
+    row_count  = csv_data.size
+    row_count -= 1 if csv_data.first&.first == '#domain'
+
+    errors.add(:data, I18n.t('imports.errors.over_rows_processing_limit', count: ROWS_PROCESSING_LIMIT)) if row_count > ROWS_PROCESSING_LIMIT
+  rescue CSV::MalformedCSVError => e
+    errors.add(:data, I18n.t('imports.errors.invalid_csv_file', error: e.message))
+  end
+end
diff --git a/app/models/domain_allow.rb b/app/models/domain_allow.rb
index 65f494fed..9e746b915 100644
--- a/app/models/domain_allow.rb
+++ b/app/models/domain_allow.rb
@@ -28,6 +28,10 @@ class DomainAllow < ApplicationRecord
       !rule_for(domain).nil?
     end
 
+    def allowed_domains
+      select(:domain)
+    end
+
     def rule_for(domain)
       return if domain.blank?
 
diff --git a/app/models/domain_block.rb b/app/models/domain_block.rb
index ad1dc2a38..8e298ac9d 100644
--- a/app/models/domain_block.rb
+++ b/app/models/domain_block.rb
@@ -29,6 +29,7 @@ class DomainBlock < ApplicationRecord
 
   scope :matches_domain, ->(value) { where(arel_table[:domain].matches("%#{value}%")) }
   scope :with_user_facing_limitations, -> { where(severity: [:silence, :suspend]) }
+  scope :with_limitations, -> { where(severity: [:silence, :suspend]).or(where(reject_media: true)) }
   scope :by_severity, -> { order(Arel.sql('(CASE severity WHEN 0 THEN 1 WHEN 1 THEN 2 WHEN 2 THEN 0 END), domain')) }
 
   def to_log_human_identifier
diff --git a/app/models/form/domain_block_batch.rb b/app/models/form/domain_block_batch.rb
new file mode 100644
index 000000000..39012df51
--- /dev/null
+++ b/app/models/form/domain_block_batch.rb
@@ -0,0 +1,35 @@
+# frozen_string_literal: true
+
+class Form::DomainBlockBatch
+  include ActiveModel::Model
+  include Authorization
+  include AccountableConcern
+
+  attr_accessor :domain_blocks_attributes, :action, :current_account
+
+  def save
+    case action
+    when 'save'
+      save!
+    end
+  end
+
+  private
+
+  def domain_blocks
+    @domain_blocks ||= domain_blocks_attributes.values.filter_map do |attributes|
+      DomainBlock.new(attributes.without('enabled')) if ActiveModel::Type::Boolean.new.cast(attributes['enabled'])
+    end
+  end
+
+  def save!
+    domain_blocks.each do |domain_block|
+      authorize(domain_block, :create?)
+      next if DomainBlock.rule_for(domain_block.domain).present?
+
+      domain_block.save!
+      DomainBlockWorker.perform_async(domain_block.id)
+      log_action :create, domain_block
+    end
+  end
+end
diff --git a/app/views/admin/export_domain_allows/new.html.haml b/app/views/admin/export_domain_allows/new.html.haml
new file mode 100644
index 000000000..dc0cf8c52
--- /dev/null
+++ b/app/views/admin/export_domain_allows/new.html.haml
@@ -0,0 +1,10 @@
+- content_for :page_title do
+  = t('.title')
+
+= simple_form_for @import, url: import_admin_export_domain_allows_path, html: { multipart: true } do |f|
+  .fields-row
+    .fields-group.fields-row__column.fields-row__column-6
+      = f.input :data, wrapper: :with_block_label, hint: t('simple_form.hints.imports.data'), as: :file
+
+  .actions
+    = f.button :button, t('imports.upload'), type: :submit
diff --git a/app/views/admin/export_domain_blocks/_domain_block.html.haml b/app/views/admin/export_domain_blocks/_domain_block.html.haml
new file mode 100644
index 000000000..5d4b6c4d0
--- /dev/null
+++ b/app/views/admin/export_domain_blocks/_domain_block.html.haml
@@ -0,0 +1,27 @@
+- existing_relationships ||= false
+
+.batch-table__row{ class: [existing_relationships && 'batch-table__row--attention'] }
+  %label.batch-table__row__select.batch-table__row__select--aligned.batch-checkbox
+    = f.check_box :enabled, checked: !existing_relationships
+  .batch-table__row__content.pending-account
+    .pending-account__header
+      %strong
+        = f.object.domain
+      = f.hidden_field :domain
+      = f.hidden_field :severity
+      = f.hidden_field :reject_media
+      = f.hidden_field :reject_reports
+      = f.hidden_field :obfuscate
+      = f.hidden_field :private_comment
+      = f.hidden_field :public_comment
+
+      %br/
+
+      = f.object.policies.map { |policy| t(policy, scope: 'admin.instances.content_policies.policies') }.join(' • ')
+      - if f.object.public_comment.present?
+        •
+        = f.object.public_comment
+      - if existing_relationships
+        •
+        = fa_icon 'warning fw'
+        = t('admin.export_domain_blocks.import.existing_relationships_warning')
diff --git a/app/views/admin/export_domain_blocks/import.html.haml b/app/views/admin/export_domain_blocks/import.html.haml
new file mode 100644
index 000000000..01add232d
--- /dev/null
+++ b/app/views/admin/export_domain_blocks/import.html.haml
@@ -0,0 +1,21 @@
+- content_for :page_title do
+  = t('admin.export_domain_blocks.import.title')
+
+%p= t('admin.export_domain_blocks.import.description_html')
+
+- if defined?(@global_private_comment) && @global_private_comment.present?
+  %p= t('admin.export_domain_blocks.import.private_comment_description_html', comment: @global_private_comment)
+
+= form_for(@form, url: batch_admin_domain_blocks_path) do |f|
+  .batch-table
+    .batch-table__toolbar
+      %label.batch-table__toolbar__select.batch-checkbox-all
+        = check_box_tag :batch_checkbox_all, nil, false
+      .batch-table__toolbar__actions
+        = f.button safe_join([fa_icon('copy'), t('admin.domain_blocks.import')]), name: :save, class: 'table-action-link', type: :submit, data: { confirm: t('admin.reports.are_you_sure') }
+    .batch-table__body
+      - if @domain_blocks.empty?
+        = nothing_here 'nothing-here--under-tabs'
+      - else
+        = f.simple_fields_for :domain_blocks, @domain_blocks do |ff|
+          = render 'domain_block', f: ff, existing_relationships: @warning_domains.include?(ff.object.domain)
diff --git a/app/views/admin/export_domain_blocks/new.html.haml b/app/views/admin/export_domain_blocks/new.html.haml
new file mode 100644
index 000000000..0291aeed7
--- /dev/null
+++ b/app/views/admin/export_domain_blocks/new.html.haml
@@ -0,0 +1,10 @@
+- content_for :page_title do
+  = t('.title')
+
+= simple_form_for @import, url: import_admin_export_domain_blocks_path, html: { multipart: true } do |f|
+  .fields-row
+    .fields-group.fields-row__column.fields-row__column-6
+      = f.input :data, wrapper: :with_block_label, hint: t('simple_form.hints.imports.data'), as: :file
+
+  .actions
+    = f.button :button, t('imports.upload'), type: :submit
diff --git a/app/views/admin/instances/index.html.haml b/app/views/admin/instances/index.html.haml
index cc5020398..8f7e3e67d 100644
--- a/app/views/admin/instances/index.html.haml
+++ b/app/views/admin/instances/index.html.haml
@@ -7,8 +7,12 @@
 - content_for :heading_actions do
   - if whitelist_mode?
     = link_to t('admin.domain_allows.add_new'), new_admin_domain_allow_path, class: 'button', id: 'add-instance-button'
+    = link_to t('admin.domain_allows.export'), export_admin_export_domain_allows_path(format: :csv), class: 'button'
+    = link_to t('admin.domain_allows.import'), new_admin_export_domain_allow_path, class: 'button'
   - else
     = link_to t('admin.domain_blocks.add_new'), new_admin_domain_block_path, class: 'button', id: 'add-instance-button'
+    = link_to t('admin.domain_blocks.export'), export_admin_export_domain_blocks_path(format: :csv), class: 'button'
+    = link_to t('admin.domain_blocks.import'), new_admin_export_domain_block_path, class: 'button'
 
 .filters
   .filter-subset