Add option to overwrite imported data (#9962)

* Add option to overwrite imported data

Fix #7465

* Add import for domain blocks

12 files changed, 148 insertions, 43 deletions

diff --git a/app/models/account_domain_block.rb b/app/models/account_domain_block.rb
index e352000c3..7c0d60379 100644
--- a/app/models/account_domain_block.rb
+++ b/app/models/account_domain_block.rb
@@ -12,6 +12,7 @@
 
 class AccountDomainBlock < ApplicationRecord
   include Paginable
+  include DomainNormalizable
 
   belongs_to :account
   validates :domain, presence: true, uniqueness: { scope: :account_id }
diff --git a/app/models/concerns/domain_normalizable.rb b/app/models/concerns/domain_normalizable.rb
index dff3e5414..fb84058fc 100644
--- a/app/models/concerns/domain_normalizable.rb
+++ b/app/models/concerns/domain_normalizable.rb
@@ -10,6 +10,6 @@ module DomainNormalizable
   private
 
   def normalize_domain
-    self.domain = TagManager.instance.normalize_domain(domain)
+    self.domain = TagManager.instance.normalize_domain(domain&.strip)
   end
 end
diff --git a/app/models/export.rb b/app/models/export.rb
index a2520e9c2..fc4bb6964 100644
--- a/app/models/export.rb
+++ b/app/models/export.rb
@@ -1,4 +1,5 @@
 # frozen_string_literal: true
+
 require 'csv'
 
 class Export
diff --git a/app/models/import.rb b/app/models/import.rb
index 55e970b0d..a7a0d8065 100644
--- a/app/models/import.rb
+++ b/app/models/import.rb
@@ -13,20 +13,30 @@
 #  data_file_size    :integer
 #  data_updated_at   :datetime
 #  account_id        :bigint(8)        not null
+#  overwrite         :boolean          default(FALSE), not null
 #
 
 class Import < ApplicationRecord
-  FILE_TYPES = ['text/plain', 'text/csv'].freeze
+  FILE_TYPES = %w(text/plain text/csv).freeze
+  MODES = %i(merge overwrite).freeze
 
   self.inheritance_column = false
 
   belongs_to :account
 
-  enum type: [:following, :blocking, :muting]
+  enum type: [:following, :blocking, :muting, :domain_blocking]
 
   validates :type, presence: true
 
   has_attached_file :data
   validates_attachment_content_type :data, content_type: FILE_TYPES
   validates_attachment_presence :data
+
+  def mode
+    overwrite? ? :overwrite : :merge
+  end
+
+  def mode=(str)
+    self.overwrite = str.to_sym == :overwrite
+  end
 end
diff --git a/app/services/import_service.rb b/app/services/import_service.rb
new file mode 100644
index 000000000..3f558626e
--- /dev/null
+++ b/app/services/import_service.rb
@@ -0,0 +1,90 @@
+# frozen_string_literal: true
+
+require 'csv'
+
+class ImportService < BaseService
+  ROWS_PROCESSING_LIMIT = 20_000
+
+  def call(import)
+    @import  = import
+    @account = @import.account
+    @data    = CSV.new(import_data).reject(&:blank?)
+
+    case @import.type
+    when 'following'
+      import_follows!
+    when 'blocking'
+      import_blocks!
+    when 'muting'
+      import_mutes!
+    when 'domain_blocking'
+      import_domain_blocks!
+    end
+  end
+
+  private
+
+  def import_follows!
+    import_relationships!('follow', 'unfollow', @account.following, follow_limit)
+  end
+
+  def import_blocks!
+    import_relationships!('block', 'unblock', @account.blocking, ROWS_PROCESSING_LIMIT)
+  end
+
+  def import_mutes!
+    import_relationships!('mute', 'unmute', @account.muting, ROWS_PROCESSING_LIMIT)
+  end
+
+  def import_domain_blocks!
+    items = @data.take(ROWS_PROCESSING_LIMIT).map { |row| row.first.strip }
+
+    if @import.overwrite?
+      presence_hash = items.each_with_object({}) { |id, mapping| mapping[id] = true }
+
+      @account.domain_blocks.find_each do |domain_block|
+        if presence_hash[domain_block.domain]
+          items.delete(domain_block.domain)
+        else
+          @account.unblock_domain!(domain_block.domain)
+        end
+      end
+    end
+
+    items.each do |domain|
+      @account.block_domain!(domain)
+    end
+
+    AfterAccountDomainBlockWorker.push_bulk(items) do |domain|
+      [@account.id, domain]
+    end
+  end
+
+  def import_relationships!(action, undo_action, overwrite_scope, limit)
+    items = @data.take(limit).map { |row| row.first.strip }
+
+    if @import.overwrite?
+      presence_hash = items.each_with_object({}) { |id, mapping| mapping[id] = true }
+
+      overwrite_scope.find_each do |target_account|
+        if presence_hash[target_account.acct]
+          items.delete(target_account.acct)
+        else
+          Import::RelationshipWorker.perform_async(@account.id, target_account.acct, undo_action)
+        end
+      end
+    end
+
+    Import::RelationshipWorker.push_bulk(items) do |acct|
+      [@account.id, acct, action]
+    end
+  end
+
+  def import_data
+    Paperclip.io_adapters.for(@import.data).read
+  end
+
+  def follow_limit
+    FollowLimitValidator.limit_for_account(@account)
+  end
+end
diff --git a/app/views/settings/imports/show.html.haml b/app/views/settings/imports/show.html.haml
index 4512fc714..7bb4beb01 100644
--- a/app/views/settings/imports/show.html.haml
+++ b/app/views/settings/imports/show.html.haml
@@ -5,8 +5,11 @@
   .field-group
     = f.input :type, collection: Import.types.keys, wrapper: :with_block_label, include_blank: false, label_method: lambda { |type| I18n.t("imports.types.#{type}") }, hint: t('imports.preface')
 
-  .field-group
-    = f.input :data, wrapper: :with_block_label, hint: t('simple_form.hints.imports.data')
+  .fields-row
+    .fields-group.fields-row__column.fields-row__column-6
+      = f.input :data, wrapper: :with_block_label, hint: t('simple_form.hints.imports.data')
+    .fields-group.fields-row__column.fields-row__column-6
+      = f.input :mode, as: :radio_buttons, collection: Import::MODES, label_method: lambda { |mode| safe_join([I18n.t("imports.modes.#{mode}"), content_tag(:span, I18n.t("imports.modes.#{mode}_long"), class: 'hint')]) }, collection_wrapper_tag: 'ul', item_wrapper_tag: 'li'
 
   .actions
     = f.button :button, t('imports.upload'), type: :submit
diff --git a/app/workers/import/relationship_worker.rb b/app/workers/import/relationship_worker.rb
index 1dd8bf8fb..e9db20a46 100644
--- a/app/workers/import/relationship_worker.rb
+++ b/app/workers/import/relationship_worker.rb
@@ -13,11 +13,17 @@ class Import::RelationshipWorker
 
     case relationship
     when 'follow'
-      FollowService.new.call(from_account, target_account.acct)
+      FollowService.new.call(from_account, target_account)
+    when 'unfollow'
+      UnfollowService.new.call(from_account, target_account)
     when 'block'
       BlockService.new.call(from_account, target_account)
+    when 'unblock'
+      UnblockService.new.call(from_account, target_account)
     when 'mute'
       MuteService.new.call(from_account, target_account)
+    when 'unmute'
+      UnmuteService.new.call(from_account, target_account)
     end
   rescue ActiveRecord::RecordNotFound
     true
diff --git a/app/workers/import_worker.rb b/app/workers/import_worker.rb
index aeb221cf6..dfa71b29e 100644
--- a/app/workers/import_worker.rb
+++ b/app/workers/import_worker.rb
@@ -1,44 +1,14 @@
 # frozen_string_literal: true
 
-require 'csv'
-
 class ImportWorker
   include Sidekiq::Worker
 
   sidekiq_options queue: 'pull', retry: false
 
-  attr_reader :import
-
   def perform(import_id)
-    @import = Import.find(import_id)
-
-    Import::RelationshipWorker.push_bulk(import_rows) do |row|
-      [@import.account_id, row.first, relationship_type]
-    end
-
-    @import.destroy
-  end
-
-  private
-
-  def import_contents
-    Paperclip.io_adapters.for(@import.data).read
-  end
-
-  def relationship_type
-    case @import.type
-    when 'following'
-      'follow'
-    when 'blocking'
-      'block'
-    when 'muting'
-      'mute'
-    end
-  end
-
-  def import_rows
-    rows = CSV.new(import_contents).reject(&:blank?)
-    rows = rows.take(FollowLimitValidator.limit_for_account(@import.account)) if @import.type == 'following'
-    rows
+    import = Import.find(import_id)
+    ImportService.new.call(import)
+  ensure
+    import&.destroy
   end
 end
diff --git a/config/locales/en.yml b/config/locales/en.yml
index c7ef36598..f23f867e5 100644
--- a/config/locales/en.yml
+++ b/config/locales/en.yml
@@ -628,10 +628,16 @@ en:
       one: Something isn't quite right yet! Please review the error below
       other: Something isn't quite right yet! Please review %{count} errors below
   imports:
+    modes:
+      merge: Merge
+      merge_long: Keep existing records and add new ones
+      overwrite: Overwrite
+      overwrite_long: Replace current records with the new ones
     preface: You can import data that you have exported from another instance, such as a list of the people you are following or blocking.
     success: Your data was successfully uploaded and will now be processed in due time
     types:
       blocking: Blocking list
+      domain_blocking: Domain blocking list
       following: Following list
       muting: Muting list
     upload: Upload
diff --git a/db/migrate/20190201012802_add_overwrite_to_imports.rb b/db/migrate/20190201012802_add_overwrite_to_imports.rb
new file mode 100644
index 000000000..89b262cc7
--- /dev/null
+++ b/db/migrate/20190201012802_add_overwrite_to_imports.rb
@@ -0,0 +1,17 @@
+require Rails.root.join('lib', 'mastodon', 'migration_helpers')
+
+class AddOverwriteToImports < ActiveRecord::Migration[5.2]
+  include Mastodon::MigrationHelpers
+
+  disable_ddl_transaction!
+
+  def up
+    safety_assured do
+      add_column_with_default :imports, :overwrite, :boolean, default: false, allow_null: false
+    end
+  end
+
+  def down
+    remove_column :imports, :overwrite, :boolean
+  end
+end
diff --git a/db/schema.rb b/db/schema.rb
index 3487adf08..44e00df40 100644
--- a/db/schema.rb
+++ b/db/schema.rb
@@ -10,7 +10,7 @@
 #
 # It's strongly recommended that you check this file into your version control system.
 
-ActiveRecord::Schema.define(version: 2019_01_17_114553) do
+ActiveRecord::Schema.define(version: 2019_02_01_012802) do
 
   # These are extensions that must be enabled in order to support this database
   enable_extension "plpgsql"
@@ -290,6 +290,7 @@ ActiveRecord::Schema.define(version: 2019_01_17_114553) do
     t.integer "data_file_size"
     t.datetime "data_updated_at"
     t.bigint "account_id", null: false
+    t.boolean "overwrite", default: false, null: false
   end
 
   create_table "invites", force: :cascade do |t|
diff --git a/spec/models/concerns/account_interactions_spec.rb b/spec/models/concerns/account_interactions_spec.rb
index 8df52b770..e8ef61f66 100644
--- a/spec/models/concerns/account_interactions_spec.rb
+++ b/spec/models/concerns/account_interactions_spec.rb
@@ -237,9 +237,9 @@ describe AccountInteractions do
   end
 
   describe '#block_domain!' do
-    let(:domain_block) { Fabricate(:domain_block) }
+    let(:domain) { 'example.com' }
 
-    subject { account.block_domain!(domain_block) }
+    subject { account.block_domain!(domain) }
 
     it 'creates and returns AccountDomainBlock' do
       expect do