diff options
| -rw-r--r-- | app/models/account_domain_block.rb | 1 | ||||
| -rw-r--r-- | app/models/concerns/domain_normalizable.rb | 2 | ||||
| -rw-r--r-- | app/models/export.rb | 1 | ||||
| -rw-r--r-- | app/models/import.rb | 14 | ||||
| -rw-r--r-- | app/services/import_service.rb | 90 | ||||
| -rw-r--r-- | app/views/settings/imports/show.html.haml | 7 | ||||
| -rw-r--r-- | app/workers/import/relationship_worker.rb | 8 | ||||
| -rw-r--r-- | app/workers/import_worker.rb | 38 | ||||
| -rw-r--r-- | config/locales/en.yml | 6 | ||||
| -rw-r--r-- | db/migrate/20190201012802_add_overwrite_to_imports.rb | 17 | ||||
| -rw-r--r-- | db/schema.rb | 3 | ||||
| -rw-r--r-- | spec/models/concerns/account_interactions_spec.rb | 4 |
12 files changed, 148 insertions, 43 deletions
diff --git a/app/models/account_domain_block.rb b/app/models/account_domain_block.rb index e352000c3..7c0d60379 100644 --- a/app/models/account_domain_block.rb +++ b/app/models/account_domain_block.rb @@ -12,6 +12,7 @@ class AccountDomainBlock < ApplicationRecord include Paginable + include DomainNormalizable belongs_to :account validates :domain, presence: true, uniqueness: { scope: :account_id } diff --git a/app/models/concerns/domain_normalizable.rb b/app/models/concerns/domain_normalizable.rb index dff3e5414..fb84058fc 100644 --- a/app/models/concerns/domain_normalizable.rb +++ b/app/models/concerns/domain_normalizable.rb @@ -10,6 +10,6 @@ module DomainNormalizable private def normalize_domain - self.domain = TagManager.instance.normalize_domain(domain) + self.domain = TagManager.instance.normalize_domain(domain&.strip) end end diff --git a/app/models/export.rb b/app/models/export.rb index a2520e9c2..fc4bb6964 100644 --- a/app/models/export.rb +++ b/app/models/export.rb @@ -1,4 +1,5 @@ # frozen_string_literal: true + require 'csv' class Export diff --git a/app/models/import.rb b/app/models/import.rb index 55e970b0d..a7a0d8065 100644 --- a/app/models/import.rb +++ b/app/models/import.rb @@ -13,20 +13,30 @@ # data_file_size :integer # data_updated_at :datetime # account_id :bigint(8) not null +# overwrite :boolean default(FALSE), not null # class Import < ApplicationRecord - FILE_TYPES = ['text/plain', 'text/csv'].freeze + FILE_TYPES = %w(text/plain text/csv).freeze + MODES = %i(merge overwrite).freeze self.inheritance_column = false belongs_to :account - enum type: [:following, :blocking, :muting] + enum type: [:following, :blocking, :muting, :domain_blocking] validates :type, presence: true has_attached_file :data validates_attachment_content_type :data, content_type: FILE_TYPES validates_attachment_presence :data + + def mode + overwrite? ? :overwrite : :merge + end + + def mode=(str) + self.overwrite = str.to_sym == :overwrite + end end diff --git a/app/services/import_service.rb b/app/services/import_service.rb new file mode 100644 index 000000000..3f558626e --- /dev/null +++ b/app/services/import_service.rb @@ -0,0 +1,90 @@ +# frozen_string_literal: true + +require 'csv' + +class ImportService < BaseService + ROWS_PROCESSING_LIMIT = 20_000 + + def call(import) + @import = import + @account = @import.account + @data = CSV.new(import_data).reject(&:blank?) + + case @import.type + when 'following' + import_follows! + when 'blocking' + import_blocks! + when 'muting' + import_mutes! + when 'domain_blocking' + import_domain_blocks! + end + end + + private + + def import_follows! + import_relationships!('follow', 'unfollow', @account.following, follow_limit) + end + + def import_blocks! + import_relationships!('block', 'unblock', @account.blocking, ROWS_PROCESSING_LIMIT) + end + + def import_mutes! + import_relationships!('mute', 'unmute', @account.muting, ROWS_PROCESSING_LIMIT) + end + + def import_domain_blocks! + items = @data.take(ROWS_PROCESSING_LIMIT).map { |row| row.first.strip } + + if @import.overwrite? + presence_hash = items.each_with_object({}) { |id, mapping| mapping[id] = true } + + @account.domain_blocks.find_each do |domain_block| + if presence_hash[domain_block.domain] + items.delete(domain_block.domain) + else + @account.unblock_domain!(domain_block.domain) + end + end + end + + items.each do |domain| + @account.block_domain!(domain) + end + + AfterAccountDomainBlockWorker.push_bulk(items) do |domain| + [@account.id, domain] + end + end + + def import_relationships!(action, undo_action, overwrite_scope, limit) + items = @data.take(limit).map { |row| row.first.strip } + + if @import.overwrite? + presence_hash = items.each_with_object({}) { |id, mapping| mapping[id] = true } + + overwrite_scope.find_each do |target_account| + if presence_hash[target_account.acct] + items.delete(target_account.acct) + else + Import::RelationshipWorker.perform_async(@account.id, target_account.acct, undo_action) + end + end + end + + Import::RelationshipWorker.push_bulk(items) do |acct| + [@account.id, acct, action] + end + end + + def import_data + Paperclip.io_adapters.for(@import.data).read + end + + def follow_limit + FollowLimitValidator.limit_for_account(@account) + end +end diff --git a/app/views/settings/imports/show.html.haml b/app/views/settings/imports/show.html.haml index 4512fc714..7bb4beb01 100644 --- a/app/views/settings/imports/show.html.haml +++ b/app/views/settings/imports/show.html.haml @@ -5,8 +5,11 @@ .field-group = f.input :type, collection: Import.types.keys, wrapper: :with_block_label, include_blank: false, label_method: lambda { |type| I18n.t("imports.types.#{type}") }, hint: t('imports.preface') - .field-group - = f.input :data, wrapper: :with_block_label, hint: t('simple_form.hints.imports.data') + .fields-row + .fields-group.fields-row__column.fields-row__column-6 + = f.input :data, wrapper: :with_block_label, hint: t('simple_form.hints.imports.data') + .fields-group.fields-row__column.fields-row__column-6 + = f.input :mode, as: :radio_buttons, collection: Import::MODES, label_method: lambda { |mode| safe_join([I18n.t("imports.modes.#{mode}"), content_tag(:span, I18n.t("imports.modes.#{mode}_long"), class: 'hint')]) }, collection_wrapper_tag: 'ul', item_wrapper_tag: 'li' .actions = f.button :button, t('imports.upload'), type: :submit diff --git a/app/workers/import/relationship_worker.rb b/app/workers/import/relationship_worker.rb index 1dd8bf8fb..e9db20a46 100644 --- a/app/workers/import/relationship_worker.rb +++ b/app/workers/import/relationship_worker.rb @@ -13,11 +13,17 @@ class Import::RelationshipWorker case relationship when 'follow' - FollowService.new.call(from_account, target_account.acct) + FollowService.new.call(from_account, target_account) + when 'unfollow' + UnfollowService.new.call(from_account, target_account) when 'block' BlockService.new.call(from_account, target_account) + when 'unblock' + UnblockService.new.call(from_account, target_account) when 'mute' MuteService.new.call(from_account, target_account) + when 'unmute' + UnmuteService.new.call(from_account, target_account) end rescue ActiveRecord::RecordNotFound true diff --git a/app/workers/import_worker.rb b/app/workers/import_worker.rb index aeb221cf6..dfa71b29e 100644 --- a/app/workers/import_worker.rb +++ b/app/workers/import_worker.rb @@ -1,44 +1,14 @@ # frozen_string_literal: true -require 'csv' - class ImportWorker include Sidekiq::Worker sidekiq_options queue: 'pull', retry: false - attr_reader :import - def perform(import_id) - @import = Import.find(import_id) - - Import::RelationshipWorker.push_bulk(import_rows) do |row| - [@import.account_id, row.first, relationship_type] - end - - @import.destroy - end - - private - - def import_contents - Paperclip.io_adapters.for(@import.data).read - end - - def relationship_type - case @import.type - when 'following' - 'follow' - when 'blocking' - 'block' - when 'muting' - 'mute' - end - end - - def import_rows - rows = CSV.new(import_contents).reject(&:blank?) - rows = rows.take(FollowLimitValidator.limit_for_account(@import.account)) if @import.type == 'following' - rows + import = Import.find(import_id) + ImportService.new.call(import) + ensure + import&.destroy end end diff --git a/config/locales/en.yml b/config/locales/en.yml index c7ef36598..f23f867e5 100644 --- a/config/locales/en.yml +++ b/config/locales/en.yml @@ -628,10 +628,16 @@ en: one: Something isn't quite right yet! Please review the error below other: Something isn't quite right yet! Please review %{count} errors below imports: + modes: + merge: Merge + merge_long: Keep existing records and add new ones + overwrite: Overwrite + overwrite_long: Replace current records with the new ones preface: You can import data that you have exported from another instance, such as a list of the people you are following or blocking. success: Your data was successfully uploaded and will now be processed in due time types: blocking: Blocking list + domain_blocking: Domain blocking list following: Following list muting: Muting list upload: Upload diff --git a/db/migrate/20190201012802_add_overwrite_to_imports.rb b/db/migrate/20190201012802_add_overwrite_to_imports.rb new file mode 100644 index 000000000..89b262cc7 --- /dev/null +++ b/db/migrate/20190201012802_add_overwrite_to_imports.rb @@ -0,0 +1,17 @@ +require Rails.root.join('lib', 'mastodon', 'migration_helpers') + +class AddOverwriteToImports < ActiveRecord::Migration[5.2] + include Mastodon::MigrationHelpers + + disable_ddl_transaction! + + def up + safety_assured do + add_column_with_default :imports, :overwrite, :boolean, default: false, allow_null: false + end + end + + def down + remove_column :imports, :overwrite, :boolean + end +end diff --git a/db/schema.rb b/db/schema.rb index 3487adf08..44e00df40 100644 --- a/db/schema.rb +++ b/db/schema.rb @@ -10,7 +10,7 @@ # # It's strongly recommended that you check this file into your version control system. -ActiveRecord::Schema.define(version: 2019_01_17_114553) do +ActiveRecord::Schema.define(version: 2019_02_01_012802) do # These are extensions that must be enabled in order to support this database enable_extension "plpgsql" @@ -290,6 +290,7 @@ ActiveRecord::Schema.define(version: 2019_01_17_114553) do t.integer "data_file_size" t.datetime "data_updated_at" t.bigint "account_id", null: false + t.boolean "overwrite", default: false, null: false end create_table "invites", force: :cascade do |t| diff --git a/spec/models/concerns/account_interactions_spec.rb b/spec/models/concerns/account_interactions_spec.rb index 8df52b770..e8ef61f66 100644 --- a/spec/models/concerns/account_interactions_spec.rb +++ b/spec/models/concerns/account_interactions_spec.rb @@ -237,9 +237,9 @@ describe AccountInteractions do end describe '#block_domain!' do - let(:domain_block) { Fabricate(:domain_block) } + let(:domain) { 'example.com' } - subject { account.block_domain!(domain_block) } + subject { account.block_domain!(domain) } it 'creates and returns AccountDomainBlock' do expect do |