From 4eeff26533b75b592395e353c6d4506db9958bcf Mon Sep 17 00:00:00 2001
From: Eugen Rochko <eugen@zeonfederated.com>
Date: Thu, 25 Jul 2019 04:17:35 +0200
Subject: Change account domain block to clear out notifications and follows
 (#11393)

---
 .../after_block_domain_from_account_service.rb     | 12 +++++++
 app/services/after_block_service.rb                | 42 +++++++---------------
 app/services/follow_service.rb                     |  2 +-
 3 files changed, 25 insertions(+), 31 deletions(-)

(limited to 'app/services')

diff --git a/app/services/after_block_domain_from_account_service.rb b/app/services/after_block_domain_from_account_service.rb
index a87c2e792..f50bde261 100644
--- a/app/services/after_block_domain_from_account_service.rb
+++ b/app/services/after_block_domain_from_account_service.rb
@@ -10,12 +10,24 @@ class AfterBlockDomainFromAccountService < BaseService
     @account = account
     @domain  = domain
 
+    clear_notifications!
+    remove_follows!
     reject_existing_followers!
     reject_pending_follow_requests!
   end
 
   private
 
+  def remove_follows!
+    @account.active_relationships.where(account: Account.where(domain: @domain)).includes(:target_account).reorder(nil).find_each do |follow|
+      UnfollowService.new.call(@account, follow.target_account)
+    end
+  end
+
+  def clear_notifications!
+    Notification.where(account: @account).where(from_account: Account.where(domain: @domain)).in_batches.delete_all
+  end
+
   def reject_existing_followers!
     @account.passive_relationships.where(account: Account.where(domain: @domain)).includes(:account).reorder(nil).find_each do |follow|
       reject_follow!(follow)
diff --git a/app/services/after_block_service.rb b/app/services/after_block_service.rb
index 706db0d63..2a0e10a79 100644
--- a/app/services/after_block_service.rb
+++ b/app/services/after_block_service.rb
@@ -2,43 +2,25 @@
 
 class AfterBlockService < BaseService
   def call(account, target_account)
-    clear_home_feed(account, target_account)
-    clear_notifications(account, target_account)
-    clear_conversations(account, target_account)
+    @account        = account
+    @target_account = target_account
+
+    clear_home_feed!
+    clear_notifications!
+    clear_conversations!
   end
 
   private
 
-  def clear_home_feed(account, target_account)
-    FeedManager.instance.clear_from_timeline(account, target_account)
+  def clear_home_feed!
+    FeedManager.instance.clear_from_timeline(@account, @target_account)
   end
 
-  def clear_conversations(account, target_account)
-    AccountConversation.where(account: account)
-                       .where('? = ANY(participant_account_ids)', target_account.id)
-                       .in_batches
-                       .destroy_all
+  def clear_conversations!
+    AccountConversation.where(account: @account).where('? = ANY(participant_account_ids)', @target_account.id).in_batches.destroy_all
   end
 
-  def clear_notifications(account, target_account)
-    Notification.where(account: account)
-                .joins(:follow)
-                .where(activity_type: 'Follow', follows: { account_id: target_account.id })
-                .delete_all
-
-    Notification.where(account: account)
-                .joins(mention: :status)
-                .where(activity_type: 'Mention', statuses: { account_id: target_account.id })
-                .delete_all
-
-    Notification.where(account: account)
-                .joins(:favourite)
-                .where(activity_type: 'Favourite', favourites: { account_id: target_account.id })
-                .delete_all
-
-    Notification.where(account: account)
-                .joins(:status)
-                .where(activity_type: 'Status', statuses: { account_id: target_account.id })
-                .delete_all
+  def clear_notifications!
+    Notification.where(account: @account).where(from_account: @target_account).in_batches.delete_all
   end
 end
diff --git a/app/services/follow_service.rb b/app/services/follow_service.rb
index 8e118f5d3..101acdaf9 100644
--- a/app/services/follow_service.rb
+++ b/app/services/follow_service.rb
@@ -13,7 +13,7 @@ class FollowService < BaseService
     target_account = ResolveAccountService.new.call(target_account, skip_webfinger: true)
 
     raise ActiveRecord::RecordNotFound if target_account.nil? || target_account.id == source_account.id || target_account.suspended?
-    raise Mastodon::NotPermittedError  if target_account.blocking?(source_account) || source_account.blocking?(target_account) || target_account.moved? || (!target_account.local? && target_account.ostatus?)
+    raise Mastodon::NotPermittedError  if target_account.blocking?(source_account) || source_account.blocking?(target_account) || target_account.moved? || (!target_account.local? && target_account.ostatus?) || source_account.domain_blocking?(target_account.domain)
 
     if source_account.following?(target_account)
       # We're already following this account, but we'll call follow! again to
-- 
cgit 


From 92569ffde81a3fe768fd8773fdab5d1a927f3f88 Mon Sep 17 00:00:00 2001
From: ThibG <thib@sitedethib.com>
Date: Fri, 26 Jul 2019 18:55:33 +0200
Subject: Fix invites not being disabled upon account suspension (#11412)

* Disable invite links from disabled/suspended users

* Add has_many invites relationship to users

* Destroy unused invites when suspending an account
---
 app/controllers/invites_controller.rb   |  2 +-
 app/models/invite.rb                    |  4 ++--
 app/models/user.rb                      |  1 +
 app/services/suspend_account_service.rb |  1 +
 spec/models/invite_spec.rb              | 16 +++++++++++-----
 5 files changed, 16 insertions(+), 8 deletions(-)

(limited to 'app/services')

diff --git a/app/controllers/invites_controller.rb b/app/controllers/invites_controller.rb
index fdb3a0962..de5280305 100644
--- a/app/controllers/invites_controller.rb
+++ b/app/controllers/invites_controller.rb
@@ -39,7 +39,7 @@ class InvitesController < ApplicationController
   private
 
   def invites
-    Invite.where(user: current_user).order(id: :desc)
+    current_user.invites.order(id: :desc)
   end
 
   def resource_params
diff --git a/app/models/invite.rb b/app/models/invite.rb
index fe2322462..02ab8e0b2 100644
--- a/app/models/invite.rb
+++ b/app/models/invite.rb
@@ -17,7 +17,7 @@
 class Invite < ApplicationRecord
   include Expireable
 
-  belongs_to :user
+  belongs_to :user, inverse_of: :invites
   has_many :users, inverse_of: :invite
 
   scope :available, -> { where(expires_at: nil).or(where('expires_at >= ?', Time.now.utc)) }
@@ -25,7 +25,7 @@ class Invite < ApplicationRecord
   before_validation :set_code
 
   def valid_for_use?
-    (max_uses.nil? || uses < max_uses) && !expired?
+    (max_uses.nil? || uses < max_uses) && !expired? && !(user.nil? || user.disabled?)
   end
 
   private
diff --git a/app/models/user.rb b/app/models/user.rb
index 474c77293..6806c0362 100644
--- a/app/models/user.rb
+++ b/app/models/user.rb
@@ -73,6 +73,7 @@ class User < ApplicationRecord
 
   has_many :applications, class_name: 'Doorkeeper::Application', as: :owner
   has_many :backups, inverse_of: :user
+  has_many :invites, inverse_of: :user
 
   has_one :invite_request, class_name: 'UserInviteRequest', inverse_of: :user, dependent: :destroy
   accepts_nested_attributes_for :invite_request, reject_if: ->(attributes) { attributes['text'].blank? }
diff --git a/app/services/suspend_account_service.rb b/app/services/suspend_account_service.rb
index 00cffcdfc..902af376c 100644
--- a/app/services/suspend_account_service.rb
+++ b/app/services/suspend_account_service.rb
@@ -64,6 +64,7 @@ class SuspendAccountService < BaseService
       @account.user.destroy
     else
       @account.user.disable!
+      @account.user.invites.where(uses: 0).destroy_all
     end
   end
 
diff --git a/spec/models/invite_spec.rb b/spec/models/invite_spec.rb
index 0ba1dccb3..30abfb86b 100644
--- a/spec/models/invite_spec.rb
+++ b/spec/models/invite_spec.rb
@@ -3,27 +3,33 @@ require 'rails_helper'
 RSpec.describe Invite, type: :model do
   describe '#valid_for_use?' do
     it 'returns true when there are no limitations' do
-      invite = Invite.new(max_uses: nil, expires_at: nil)
+      invite = Fabricate(:invite, max_uses: nil, expires_at: nil)
       expect(invite.valid_for_use?).to be true
     end
 
     it 'returns true when not expired' do
-      invite = Invite.new(max_uses: nil, expires_at: 1.hour.from_now)
+      invite = Fabricate(:invite, max_uses: nil, expires_at: 1.hour.from_now)
       expect(invite.valid_for_use?).to be true
     end
 
     it 'returns false when expired' do
-      invite = Invite.new(max_uses: nil, expires_at: 1.hour.ago)
+      invite = Fabricate(:invite, max_uses: nil, expires_at: 1.hour.ago)
       expect(invite.valid_for_use?).to be false
     end
 
     it 'returns true when uses still available' do
-      invite = Invite.new(max_uses: 250, uses: 249, expires_at: nil)
+      invite = Fabricate(:invite, max_uses: 250, uses: 249, expires_at: nil)
       expect(invite.valid_for_use?).to be true
     end
 
     it 'returns false when maximum uses reached' do
-      invite = Invite.new(max_uses: 250, uses: 250, expires_at: nil)
+      invite = Fabricate(:invite, max_uses: 250, uses: 250, expires_at: nil)
+      expect(invite.valid_for_use?).to be false
+    end
+
+    it 'returns false when invite creator has been disabled' do
+      invite = Fabricate(:invite, max_uses: nil, expires_at: nil)
+      SuspendAccountService.new.call(invite.user.account)
       expect(invite.valid_for_use?).to be false
     end
   end
-- 
cgit 


From b9fbcbfe4e0a15fcf8a457ce17ea080f0eb939fc Mon Sep 17 00:00:00 2001
From: Eugen Rochko <eugen@zeonfederated.com>
Date: Sat, 27 Jul 2019 04:42:08 +0200
Subject: Add search syntax for operators and phrases (#11411)

---
 Gemfile                              |  1 +
 Gemfile.lock                         |  2 +
 app/lib/search_query_parser.rb       | 14 ++++++
 app/lib/search_query_transformer.rb  | 86 ++++++++++++++++++++++++++++++++++++
 app/services/search_service.rb       |  9 ++--
 spec/services/search_service_spec.rb |  6 +--
 6 files changed, 112 insertions(+), 6 deletions(-)
 create mode 100644 app/lib/search_query_parser.rb
 create mode 100644 app/lib/search_query_transformer.rb

(limited to 'app/services')

diff --git a/Gemfile b/Gemfile
index e969c4ae0..96eb44af7 100644
--- a/Gemfile
+++ b/Gemfile
@@ -64,6 +64,7 @@ gem 'nsa', '~> 0.2'
 gem 'oj', '~> 3.8'
 gem 'ostatus2', '~> 2.0'
 gem 'ox', '~> 2.11'
+gem 'parslet'
 gem 'posix-spawn', git: 'https://github.com/rtomayko/posix-spawn', ref: '58465d2e213991f8afb13b984854a49fcdcc980c'
 gem 'pundit', '~> 2.0'
 gem 'premailer-rails'
diff --git a/Gemfile.lock b/Gemfile.lock
index f435b3a35..5adae6f9c 100644
--- a/Gemfile.lock
+++ b/Gemfile.lock
@@ -404,6 +404,7 @@ GEM
       parallel
     parser (2.6.3.0)
       ast (~> 2.4.0)
+    parslet (1.8.2)
     pastel (0.7.2)
       equatable (~> 0.5.0)
       tty-color (~> 0.4.0)
@@ -724,6 +725,7 @@ DEPENDENCIES
   paperclip (~> 6.0)
   paperclip-av-transcoder (~> 0.6)
   parallel_tests (~> 2.29)
+  parslet
   pg (~> 1.1)
   pghero (~> 2.2)
   pkg-config (~> 1.3)
diff --git a/app/lib/search_query_parser.rb b/app/lib/search_query_parser.rb
new file mode 100644
index 000000000..405ad15b8
--- /dev/null
+++ b/app/lib/search_query_parser.rb
@@ -0,0 +1,14 @@
+# frozen_string_literal: true
+
+class SearchQueryParser < Parslet::Parser
+  rule(:term)     { match('[^\s":]').repeat(1).as(:term) }
+  rule(:quote)    { str('"') }
+  rule(:colon)    { str(':') }
+  rule(:space)    { match('\s').repeat(1) }
+  rule(:operator) { (str('+') | str('-')).as(:operator) }
+  rule(:prefix)   { (term >> colon).as(:prefix) }
+  rule(:phrase)   { (quote >> (term >> space.maybe).repeat >> quote).as(:phrase) }
+  rule(:clause)   { (prefix.maybe >> operator.maybe >> (phrase | term)).as(:clause) }
+  rule(:query)    { (clause >> space.maybe).repeat.as(:query) }
+  root(:query)
+end
diff --git a/app/lib/search_query_transformer.rb b/app/lib/search_query_transformer.rb
new file mode 100644
index 000000000..2c4144790
--- /dev/null
+++ b/app/lib/search_query_transformer.rb
@@ -0,0 +1,86 @@
+# frozen_string_literal: true
+
+class SearchQueryTransformer < Parslet::Transform
+  class Query
+    attr_reader :should_clauses, :must_not_clauses, :must_clauses
+
+    def initialize(clauses)
+      grouped = clauses.chunk(&:operator).to_h
+      @should_clauses = grouped.fetch(:should, [])
+      @must_not_clauses = grouped.fetch(:must_not, [])
+      @must_clauses = grouped.fetch(:must, [])
+    end
+
+    def apply(search)
+      should_clauses.each { |clause| search = search.query.should(clause_to_query(clause)) }
+      must_clauses.each { |clause| search = search.query.must(clause_to_query(clause)) }
+      must_not_clauses.each { |clause| search = search.query.must_not(clause_to_query(clause)) }
+      search.query.minimum_should_match(1)
+    end
+
+    private
+
+    def clause_to_query(clause)
+      case clause
+      when TermClause
+        { multi_match: { type: 'most_fields', query: clause.term, fields: ['text', 'text.stemmed'] } }
+      when PhraseClause
+        { match_phrase: { text: { query: clause.phrase } } }
+      else
+        raise "Unexpected clause type: #{clause}"
+      end
+    end
+  end
+
+  class Operator
+    class << self
+      def symbol(str)
+        case str
+        when '+'
+          :must
+        when '-'
+          :must_not
+        when nil
+          :should
+        else
+          raise "Unknown operator: #{str}"
+        end
+      end
+    end
+  end
+
+  class TermClause
+    attr_reader :prefix, :operator, :term
+
+    def initialize(prefix, operator, term)
+      @prefix = prefix
+      @operator = Operator.symbol(operator)
+      @term = term
+    end
+  end
+
+  class PhraseClause
+    attr_reader :prefix, :operator, :phrase
+
+    def initialize(prefix, operator, phrase)
+      @prefix = prefix
+      @operator = Operator.symbol(operator)
+      @phrase = phrase
+    end
+  end
+
+  rule(clause: subtree(:clause)) do
+    prefix   = clause[:prefix][:term].to_s if clause[:prefix]
+    operator = clause[:operator]&.to_s
+
+    if clause[:term]
+      TermClause.new(prefix, operator, clause[:term].to_s)
+    elsif clause[:phrase]
+      PhraseClause.new(prefix, operator, clause[:phrase].map { |p| p[:term].to_s }.join(' '))
+    else
+      raise "Unexpected clause type: #{clause}"
+    end
+  end
+
+  rule(query: sequence(:clauses)) { Query.new(clauses) }
+end
diff --git a/app/services/search_service.rb b/app/services/search_service.rb
index e0da61dac..769d1ac7a 100644
--- a/app/services/search_service.rb
+++ b/app/services/search_service.rb
@@ -33,8 +33,7 @@ class SearchService < BaseService
   end
 
   def perform_statuses_search!
-    definition = StatusesIndex.filter(term: { searchable_by: @account.id })
-                              .query(multi_match: { type: 'most_fields', query: @query, operator: 'and', fields: %w(text text.stemmed) })
+    definition = parsed_query.apply(StatusesIndex.filter(term: { searchable_by: @account.id }))
 
     if @options[:account_id].present?
       definition = definition.filter(term: { account_id: @options[:account_id] })
@@ -70,7 +69,7 @@ class SearchService < BaseService
   end
 
   def url_query?
-    @options[:type].blank? && @query =~ /\Ahttps?:\/\//
+    @resolve && @options[:type].blank? && @query =~ /\Ahttps?:\/\//
   end
 
   def url_resource_results
@@ -120,4 +119,8 @@ class SearchService < BaseService
       domain_blocking_by_domain: Account.domain_blocking_map_by_domain(domains, account.id),
     }
   end
+
+  def parsed_query
+    SearchQueryTransformer.new.apply(SearchQueryParser.new.parse(@query))
+  end
 end
diff --git a/spec/services/search_service_spec.rb b/spec/services/search_service_spec.rb
index d064cd9b8..ade306ed2 100644
--- a/spec/services/search_service_spec.rb
+++ b/spec/services/search_service_spec.rb
@@ -27,7 +27,7 @@ describe SearchService, type: :service do
         it 'returns the empty results' do
           service = double(call: nil)
           allow(ResolveURLService).to receive(:new).and_return(service)
-          results = subject.call(@query, nil, 10)
+          results = subject.call(@query, nil, 10, resolve: true)
 
           expect(service).to have_received(:call).with(@query, on_behalf_of: nil)
           expect(results).to eq empty_results
@@ -40,7 +40,7 @@ describe SearchService, type: :service do
           service = double(call: account)
           allow(ResolveURLService).to receive(:new).and_return(service)
 
-          results = subject.call(@query, nil, 10)
+          results = subject.call(@query, nil, 10, resolve: true)
           expect(service).to have_received(:call).with(@query, on_behalf_of: nil)
           expect(results).to eq empty_results.merge(accounts: [account])
         end
@@ -52,7 +52,7 @@ describe SearchService, type: :service do
           service = double(call: status)
           allow(ResolveURLService).to receive(:new).and_return(service)
 
-          results = subject.call(@query, nil, 10)
+          results = subject.call(@query, nil, 10, resolve: true)
           expect(service).to have_received(:call).with(@query, on_behalf_of: nil)
           expect(results).to eq empty_results.merge(statuses: [status])
         end
-- 
cgit 


From f371b32137ccd7e74ca29d25af2072fb79654b15 Mon Sep 17 00:00:00 2001
From: Eugen Rochko <eugen@zeonfederated.com>
Date: Sun, 28 Jul 2019 05:59:51 +0200
Subject: Change hashtags to preserve first-used casing (#11416)

---
 app/lib/activitypub/activity/create.rb             |  9 ++----
 app/models/tag.rb                                  | 34 +++++++++++++++++++---
 app/services/hashtag_query_service.rb              |  4 +--
 app/services/process_hashtags_service.rb           |  4 +--
 ...726175042_add_case_insensitive_index_to_tags.rb | 15 ++++++++++
 db/schema.rb                                       |  5 ++--
 6 files changed, 53 insertions(+), 18 deletions(-)
 create mode 100644 db/migrate/20190726175042_add_case_insensitive_index_to_tags.rb

(limited to 'app/services')

diff --git a/app/lib/activitypub/activity/create.rb b/app/lib/activitypub/activity/create.rb
index 56c24680a..000b77df5 100644
--- a/app/lib/activitypub/activity/create.rb
+++ b/app/lib/activitypub/activity/create.rb
@@ -148,12 +148,9 @@ class ActivityPub::Activity::Create < ActivityPub::Activity
   def process_hashtag(tag)
     return if tag['name'].blank?
 
-    hashtag = tag['name'].gsub(/\A#/, '').mb_chars.downcase
-    hashtag = Tag.where(name: hashtag).first_or_create!(name: hashtag)
-
-    return if @tags.include?(hashtag)
-
-    @tags << hashtag
+    Tag.find_or_create_by_names(tag['name']) do |hashtag|
+      @tags << hashtag unless @tags.include?(hashtag)
+    end
   rescue ActiveRecord::RecordInvalid
     nil
   end
diff --git a/app/models/tag.rb b/app/models/tag.rb
index b371d59c1..972242064 100644
--- a/app/models/tag.rb
+++ b/app/models/tag.rb
@@ -20,7 +20,7 @@ class Tag < ApplicationRecord
   HASHTAG_NAME_RE = '([[:word:]_][[:word:]_·]*[[:alpha:]_·][[:word:]_·]*[[:word:]_])|([[:word:]_]*[[:alpha:]][[:word:]_]*)'
   HASHTAG_RE = /(?:^|[^\/\)\w])#(#{HASHTAG_NAME_RE})/i
 
-  validates :name, presence: true, uniqueness: true, format: { with: /\A(#{HASHTAG_NAME_RE})\z/i }
+  validates :name, presence: true, format: { with: /\A(#{HASHTAG_NAME_RE})\z/i }
 
   scope :discoverable, -> { joins(:account_tag_stat).where(AccountTagStat.arel_table[:accounts_count].gt(0)).where(account_tag_stats: { hidden: false }).order(Arel.sql('account_tag_stats.accounts_count desc')) }
   scope :hidden, -> { where(account_tag_stats: { hidden: true }) }
@@ -64,22 +64,48 @@ class Tag < ApplicationRecord
   end
 
   class << self
+    def find_or_create_by_names(name_or_names)
+      Array(name_or_names).map(&method(:normalize)).uniq.map do |normalized_name|
+        tag = matching_name(normalized_name).first || create(name: normalized_name)
+
+        yield tag if block_given?
+
+        tag
+      end
+    end
+
     def search_for(term, limit = 5, offset = 0)
-      pattern = sanitize_sql_like(term.strip) + '%'
+      pattern = sanitize_sql_like(normalize(term.strip)) + '%'
 
-      Tag.where('lower(name) like lower(?)', pattern)
+      Tag.where(arel_table[:name].lower.matches(pattern.downcase))
          .order(:name)
          .limit(limit)
          .offset(offset)
     end
 
     def find_normalized(name)
-      find_by(name: name.mb_chars.downcase.to_s)
+      matching_name(name).first
     end
 
     def find_normalized!(name)
       find_normalized(name) || raise(ActiveRecord::RecordNotFound)
     end
+
+    def matching_name(name_or_names)
+      names = Array(name_or_names).map { |name| normalize(name).downcase }
+
+      if names.size == 1
+        where(arel_table[:name].lower.eq(names.first))
+      else
+        where(arel_table[:name].lower.in(names))
+      end
+    end
+
+    private
+
+    def normalize(str)
+      str.gsub(/\A#/, '').mb_chars.to_s
+    end
   end
 
   private
diff --git a/app/services/hashtag_query_service.rb b/app/services/hashtag_query_service.rb
index 5773d78c6..282821710 100644
--- a/app/services/hashtag_query_service.rb
+++ b/app/services/hashtag_query_service.rb
@@ -14,7 +14,7 @@ class HashtagQueryService < BaseService
 
   private
 
-  def tags_for(tags)
-    Tag.where(name: tags.map(&:downcase)) if tags.presence
+  def tags_for(names)
+    Tag.matching_name(names) if names.presence
   end
 end
diff --git a/app/services/process_hashtags_service.rb b/app/services/process_hashtags_service.rb
index b6974e598..e8e139b05 100644
--- a/app/services/process_hashtags_service.rb
+++ b/app/services/process_hashtags_service.rb
@@ -5,9 +5,7 @@ class ProcessHashtagsService < BaseService
     tags    = Extractor.extract_hashtags(status.text) if status.local?
     records = []
 
-    tags.map { |str| str.mb_chars.downcase }.uniq(&:to_s).each do |name|
-      tag = Tag.where(name: name).first_or_create(name: name)
-
+    Tag.find_or_create_by_names(tags) do |tag|
       status.tags << tag
       records << tag
 
diff --git a/db/migrate/20190726175042_add_case_insensitive_index_to_tags.rb b/db/migrate/20190726175042_add_case_insensitive_index_to_tags.rb
new file mode 100644
index 000000000..6fa8c0ec4
--- /dev/null
+++ b/db/migrate/20190726175042_add_case_insensitive_index_to_tags.rb
@@ -0,0 +1,15 @@
+class AddCaseInsensitiveIndexToTags < ActiveRecord::Migration[5.2]
+  disable_ddl_transaction!
+
+  def up
+    safety_assured { execute 'CREATE UNIQUE INDEX CONCURRENTLY index_tags_on_name_lower ON tags (lower(name))' }
+    remove_index :tags, name: 'index_tags_on_name'
+    remove_index :tags, name: 'hashtag_search_index'
+  end
+
+  def down
+    add_index :tags, :name, unique: true, algorithm: :concurrently
+    safety_assured { execute 'CREATE INDEX CONCURRENTLY hashtag_search_index ON tags (name text_pattern_ops)' }
+    remove_index :tags, name: 'index_tags_on_name_lower'
+  end
+end
diff --git a/db/schema.rb b/db/schema.rb
index 6319dd932..1847305c7 100644
--- a/db/schema.rb
+++ b/db/schema.rb
@@ -10,7 +10,7 @@
 #
 # It's strongly recommended that you check this file into your version control system.
 
-ActiveRecord::Schema.define(version: 2019_07_15_164535) do
+ActiveRecord::Schema.define(version: 2019_07_26_175042) do
 
   # These are extensions that must be enabled in order to support this database
   enable_extension "plpgsql"
@@ -652,8 +652,7 @@ ActiveRecord::Schema.define(version: 2019_07_15_164535) do
     t.string "name", default: "", null: false
     t.datetime "created_at", null: false
     t.datetime "updated_at", null: false
-    t.index "lower((name)::text) text_pattern_ops", name: "hashtag_search_index"
-    t.index ["name"], name: "index_tags_on_name", unique: true
+    t.index "lower((name)::text)", name: "index_tags_on_name_lower", unique: true
   end
 
   create_table "tombstones", force: :cascade do |t|
-- 
cgit 


From 24552b5160a5090e7d6056fb69a209aa48fe4fce Mon Sep 17 00:00:00 2001
From: Eugen Rochko <eugen@zeonfederated.com>
Date: Tue, 30 Jul 2019 11:10:46 +0200
Subject: Add whitelist mode (#11291)

---
 app/controllers/about_controller.rb                |  5 +++
 app/controllers/activitypub/base_controller.rb     |  2 ++
 app/controllers/activitypub/inboxes_controller.rb  |  2 +-
 app/controllers/admin/domain_allows_controller.rb  | 40 ++++++++++++++++++++++
 app/controllers/admin/instances_controller.rb      | 28 +++++++++++++--
 app/controllers/api/base_controller.rb             |  9 +++++
 app/controllers/api/v1/accounts_controller.rb      |  2 ++
 app/controllers/api/v1/apps_controller.rb          |  2 ++
 .../api/v1/instances/activity_controller.rb        |  3 +-
 .../api/v1/instances/peers_controller.rb           |  3 +-
 app/controllers/api/v1/instances_controller.rb     |  1 +
 app/controllers/application_controller.rb          |  4 ++-
 app/controllers/concerns/account_owned_concern.rb  |  1 +
 app/controllers/directories_controller.rb          |  5 +--
 app/controllers/home_controller.rb                 |  2 +-
 app/controllers/media_controller.rb                |  1 +
 app/controllers/media_proxy_controller.rb          |  2 ++
 app/controllers/public_timelines_controller.rb     |  5 +--
 app/controllers/remote_interaction_controller.rb   |  1 +
 app/controllers/tags_controller.rb                 |  1 +
 app/helpers/domain_control_helper.rb               | 10 +++++-
 app/models/domain_allow.rb                         | 33 ++++++++++++++++++
 app/models/instance.rb                             |  3 +-
 app/models/instance_filter.rb                      |  4 +++
 app/policies/domain_allow_policy.rb                | 11 ++++++
 app/services/concerns/payloadable.rb               |  2 +-
 app/services/unallow_domain_service.rb             | 11 ++++++
 app/views/admin/domain_allows/new.html.haml        | 14 ++++++++
 app/views/admin/instances/index.html.haml          | 35 ++++++++++++-------
 app/views/admin/instances/show.html.haml           |  4 ++-
 app/views/admin/settings/edit.html.haml            | 28 ++++++++-------
 app/views/auth/registrations/new.html.haml         |  2 +-
 app/views/layouts/public.html.haml                 |  9 +++--
 config/initializers/2_whitelist_mode.rb            |  5 +++
 config/locales/en.yml                              |  7 ++++
 config/locales/simple_form.en.yml                  |  2 ++
 config/navigation.rb                               |  2 +-
 config/routes.rb                                   |  1 +
 db/migrate/20190705002136_create_domain_allows.rb  |  9 +++++
 db/schema.rb                                       |  9 ++++-
 lib/mastodon/domains_cli.rb                        | 22 ++++++++++--
 spec/fabricators/domain_allow_fabricator.rb        |  3 ++
 spec/models/domain_allow_spec.rb                   |  5 +++
 streaming/index.js                                 |  5 +--
 44 files changed, 302 insertions(+), 53 deletions(-)
 create mode 100644 app/controllers/admin/domain_allows_controller.rb
 create mode 100644 app/models/domain_allow.rb
 create mode 100644 app/policies/domain_allow_policy.rb
 create mode 100644 app/services/unallow_domain_service.rb
 create mode 100644 app/views/admin/domain_allows/new.html.haml
 create mode 100644 config/initializers/2_whitelist_mode.rb
 create mode 100644 db/migrate/20190705002136_create_domain_allows.rb
 create mode 100644 spec/fabricators/domain_allow_fabricator.rb
 create mode 100644 spec/models/domain_allow_spec.rb

(limited to 'app/services')

diff --git a/app/controllers/about_controller.rb b/app/controllers/about_controller.rb
index 31cf17710..d276e8fe5 100644
--- a/app/controllers/about_controller.rb
+++ b/app/controllers/about_controller.rb
@@ -3,6 +3,7 @@
 class AboutController < ApplicationController
   layout 'public'
 
+  before_action :require_open_federation!, only: [:show, :more]
   before_action :set_body_classes, only: :show
   before_action :set_instance_presenter
   before_action :set_expires_in
@@ -19,6 +20,10 @@ class AboutController < ApplicationController
 
   private
 
+  def require_open_federation!
+    not_found if whitelist_mode?
+  end
+
   def new_user
     User.new.tap do |user|
       user.build_account
diff --git a/app/controllers/activitypub/base_controller.rb b/app/controllers/activitypub/base_controller.rb
index a3b5c4dfa..0c2591e97 100644
--- a/app/controllers/activitypub/base_controller.rb
+++ b/app/controllers/activitypub/base_controller.rb
@@ -1,6 +1,8 @@
 # frozen_string_literal: true
 
 class ActivityPub::BaseController < Api::BaseController
+  skip_before_action :require_authenticated_user!
+
   private
 
   def set_cache_headers
diff --git a/app/controllers/activitypub/inboxes_controller.rb b/app/controllers/activitypub/inboxes_controller.rb
index 7cfd9a25e..bcfc1e6d4 100644
--- a/app/controllers/activitypub/inboxes_controller.rb
+++ b/app/controllers/activitypub/inboxes_controller.rb
@@ -1,6 +1,6 @@
 # frozen_string_literal: true
 
-class ActivityPub::InboxesController < Api::BaseController
+class ActivityPub::InboxesController < ActivityPub::BaseController
   include SignatureVerification
   include JsonLdHelper
   include AccountOwnedConcern
diff --git a/app/controllers/admin/domain_allows_controller.rb b/app/controllers/admin/domain_allows_controller.rb
new file mode 100644
index 000000000..31be1978b
--- /dev/null
+++ b/app/controllers/admin/domain_allows_controller.rb
@@ -0,0 +1,40 @@
+# frozen_string_literal: true
+
+class Admin::DomainAllowsController < Admin::BaseController
+  before_action :set_domain_allow, only: [:destroy]
+
+  def new
+    authorize :domain_allow, :create?
+
+    @domain_allow = DomainAllow.new(domain: params[:_domain])
+  end
+
+  def create
+    authorize :domain_allow, :create?
+
+    @domain_allow = DomainAllow.new(resource_params)
+
+    if @domain_allow.save
+      log_action :create, @domain_allow
+      redirect_to admin_instances_path, notice: I18n.t('admin.domain_allows.created_msg')
+    else
+      render :new
+    end
+  end
+
+  def destroy
+    authorize @domain_allow, :destroy?
+    UnallowDomainService.new.call(@domain_allow)
+    redirect_to admin_instances_path, notice: I18n.t('admin.domain_allows.destroyed_msg')
+  end
+
+  private
+
+  def set_domain_allow
+    @domain_allow = DomainAllow.find(params[:id])
+  end
+
+  def resource_params
+    params.require(:domain_allow).permit(:domain)
+  end
+end
diff --git a/app/controllers/admin/instances_controller.rb b/app/controllers/admin/instances_controller.rb
index 7888e844f..d4f201807 100644
--- a/app/controllers/admin/instances_controller.rb
+++ b/app/controllers/admin/instances_controller.rb
@@ -2,6 +2,10 @@
 
 module Admin
   class InstancesController < BaseController
+    before_action :set_domain_block, only: :show
+    before_action :set_domain_allow, only: :show
+    before_action :set_instance, only: :show
+
     def index
       authorize :instance, :index?
 
@@ -11,20 +15,38 @@ module Admin
     def show
       authorize :instance, :show?
 
-      @instance        = Instance.new(Account.by_domain_accounts.find_by(domain: params[:id]) || DomainBlock.find_by!(domain: params[:id]))
       @following_count = Follow.where(account: Account.where(domain: params[:id])).count
       @followers_count = Follow.where(target_account: Account.where(domain: params[:id])).count
       @reports_count   = Report.where(target_account: Account.where(domain: params[:id])).count
       @blocks_count    = Block.where(target_account: Account.where(domain: params[:id])).count
       @available       = DeliveryFailureTracker.available?(Account.select(:shared_inbox_url).where(domain: params[:id]).first&.shared_inbox_url)
       @media_storage   = MediaAttachment.where(account: Account.where(domain: params[:id])).sum(:file_file_size)
-      @domain_block    = DomainBlock.rule_for(params[:id])
     end
 
     private
 
+    def set_domain_block
+      @domain_block = DomainBlock.rule_for(params[:id])
+    end
+
+    def set_domain_allow
+      @domain_allow = DomainAllow.rule_for(params[:id])
+    end
+
+    def set_instance
+      resource   = Account.by_domain_accounts.find_by(domain: params[:id])
+      resource ||= @domain_block
+      resource ||= @domain_allow
+
+      if resource
+        @instance = Instance.new(resource)
+      else
+        not_found
+      end
+    end
+
     def filtered_instances
-      InstanceFilter.new(filter_params).results
+      InstanceFilter.new(whitelist_mode? ? { allowed: true } : filter_params).results
     end
 
     def paginated_instances
diff --git a/app/controllers/api/base_controller.rb b/app/controllers/api/base_controller.rb
index 6f33a1ea9..109e38ffa 100644
--- a/app/controllers/api/base_controller.rb
+++ b/app/controllers/api/base_controller.rb
@@ -9,6 +9,7 @@ class Api::BaseController < ApplicationController
   skip_before_action :store_current_location
   skip_before_action :require_functional!
 
+  before_action :require_authenticated_user!, if: :disallow_unauthenticated_api_access?
   before_action :set_cache_headers
 
   protect_from_forgery with: :null_session
@@ -69,6 +70,10 @@ class Api::BaseController < ApplicationController
     nil
   end
 
+  def require_authenticated_user!
+    render json: { error: 'This API requires an authenticated user' }, status: 401 unless current_user
+  end
+
   def require_user!
     if !current_user
       render json: { error: 'This method requires an authenticated user' }, status: 422
@@ -94,4 +99,8 @@ class Api::BaseController < ApplicationController
   def set_cache_headers
     response.headers['Cache-Control'] = 'no-cache, no-store, max-age=0, must-revalidate'
   end
+
+  def disallow_unauthenticated_api_access?
+    authorized_fetch_mode?
+  end
 end
diff --git a/app/controllers/api/v1/accounts_controller.rb b/app/controllers/api/v1/accounts_controller.rb
index b0c62778e..b306e8e8c 100644
--- a/app/controllers/api/v1/accounts_controller.rb
+++ b/app/controllers/api/v1/accounts_controller.rb
@@ -12,6 +12,8 @@ class Api::V1::AccountsController < Api::BaseController
   before_action :check_account_suspension, only: [:show]
   before_action :check_enabled_registrations, only: [:create]
 
+  skip_before_action :require_authenticated_user!, only: :create
+
   respond_to :json
 
   def show
diff --git a/app/controllers/api/v1/apps_controller.rb b/app/controllers/api/v1/apps_controller.rb
index e9f7a7291..97177547a 100644
--- a/app/controllers/api/v1/apps_controller.rb
+++ b/app/controllers/api/v1/apps_controller.rb
@@ -1,6 +1,8 @@
 # frozen_string_literal: true
 
 class Api::V1::AppsController < Api::BaseController
+  skip_before_action :require_authenticated_user!
+
   def create
     @app = Doorkeeper::Application.create!(application_options)
     render json: @app, serializer: REST::ApplicationSerializer
diff --git a/app/controllers/api/v1/instances/activity_controller.rb b/app/controllers/api/v1/instances/activity_controller.rb
index d0080c5c2..4fb5a69d8 100644
--- a/app/controllers/api/v1/instances/activity_controller.rb
+++ b/app/controllers/api/v1/instances/activity_controller.rb
@@ -2,6 +2,7 @@
 
 class Api::V1::Instances::ActivityController < Api::BaseController
   before_action :require_enabled_api!
+
   skip_before_action :set_cache_headers
 
   respond_to :json
@@ -33,6 +34,6 @@ class Api::V1::Instances::ActivityController < Api::BaseController
   end
 
   def require_enabled_api!
-    head 404 unless Setting.activity_api_enabled
+    head 404 unless Setting.activity_api_enabled && !whitelist_mode?
   end
 end
diff --git a/app/controllers/api/v1/instances/peers_controller.rb b/app/controllers/api/v1/instances/peers_controller.rb
index 450e6502f..75c3cb4ba 100644
--- a/app/controllers/api/v1/instances/peers_controller.rb
+++ b/app/controllers/api/v1/instances/peers_controller.rb
@@ -2,6 +2,7 @@
 
 class Api::V1::Instances::PeersController < Api::BaseController
   before_action :require_enabled_api!
+
   skip_before_action :set_cache_headers
 
   respond_to :json
@@ -14,6 +15,6 @@ class Api::V1::Instances::PeersController < Api::BaseController
   private
 
   def require_enabled_api!
-    head 404 unless Setting.peers_api_enabled
+    head 404 unless Setting.peers_api_enabled && !whitelist_mode?
   end
 end
diff --git a/app/controllers/api/v1/instances_controller.rb b/app/controllers/api/v1/instances_controller.rb
index 93e4f0003..8d8231423 100644
--- a/app/controllers/api/v1/instances_controller.rb
+++ b/app/controllers/api/v1/instances_controller.rb
@@ -2,6 +2,7 @@
 
 class Api::V1::InstancesController < Api::BaseController
   respond_to :json
+
   skip_before_action :set_cache_headers
 
   def show
diff --git a/app/controllers/application_controller.rb b/app/controllers/application_controller.rb
index 41ce1a0ca..0d3913ee0 100644
--- a/app/controllers/application_controller.rb
+++ b/app/controllers/application_controller.rb
@@ -11,12 +11,14 @@ class ApplicationController < ActionController::Base
   include UserTrackingConcern
   include SessionTrackingConcern
   include CacheConcern
+  include DomainControlHelper
 
   helper_method :current_account
   helper_method :current_session
   helper_method :current_theme
   helper_method :single_user_mode?
   helper_method :use_seamless_external_login?
+  helper_method :whitelist_mode?
 
   rescue_from ActionController::RoutingError, with: :not_found
   rescue_from ActiveRecord::RecordNotFound, with: :not_found
@@ -38,7 +40,7 @@ class ApplicationController < ActionController::Base
   end
 
   def authorized_fetch_mode?
-    ENV['AUTHORIZED_FETCH'] == 'true'
+    ENV['AUTHORIZED_FETCH'] == 'true' || Rails.configuration.x.whitelist_mode
   end
 
   def public_fetch_mode?
diff --git a/app/controllers/concerns/account_owned_concern.rb b/app/controllers/concerns/account_owned_concern.rb
index 99c240fe9..460f71f65 100644
--- a/app/controllers/concerns/account_owned_concern.rb
+++ b/app/controllers/concerns/account_owned_concern.rb
@@ -4,6 +4,7 @@ module AccountOwnedConcern
   extend ActiveSupport::Concern
 
   included do
+    before_action :authenticate_user!, if: -> { whitelist_mode? && request.format != :json }
     before_action :set_account, if: :account_required?
     before_action :check_account_approval, if: :account_required?
     before_action :check_account_suspension, if: :account_required?
diff --git a/app/controllers/directories_controller.rb b/app/controllers/directories_controller.rb
index 594907674..d2ef76f06 100644
--- a/app/controllers/directories_controller.rb
+++ b/app/controllers/directories_controller.rb
@@ -3,7 +3,8 @@
 class DirectoriesController < ApplicationController
   layout 'public'
 
-  before_action :check_enabled
+  before_action :authenticate_user!, if: :whitelist_mode?
+  before_action :require_enabled!
   before_action :set_instance_presenter
   before_action :set_tag, only: :show
   before_action :set_tags
@@ -19,7 +20,7 @@ class DirectoriesController < ApplicationController
 
   private
 
-  def check_enabled
+  def require_enabled!
     return not_found unless Setting.profile_directory
   end
 
diff --git a/app/controllers/home_controller.rb b/app/controllers/home_controller.rb
index 42493cd78..22d507e77 100644
--- a/app/controllers/home_controller.rb
+++ b/app/controllers/home_controller.rb
@@ -55,7 +55,7 @@ class HomeController < ApplicationController
   end
 
   def default_redirect_path
-    if request.path.start_with?('/web')
+    if request.path.start_with?('/web') || whitelist_mode?
       new_user_session_path
     elsif single_user_mode?
       short_account_path(Account.local.without_suspended.where('id > 0').first)
diff --git a/app/controllers/media_controller.rb b/app/controllers/media_controller.rb
index b3b7519a1..1f693de32 100644
--- a/app/controllers/media_controller.rb
+++ b/app/controllers/media_controller.rb
@@ -5,6 +5,7 @@ class MediaController < ApplicationController
 
   skip_before_action :store_current_location
 
+  before_action :authenticate_user!, if: :whitelist_mode?
   before_action :set_media_attachment
   before_action :verify_permitted_status!
   before_action :check_playable, only: :player
diff --git a/app/controllers/media_proxy_controller.rb b/app/controllers/media_proxy_controller.rb
index 8fc18dd06..8da6c6fe0 100644
--- a/app/controllers/media_proxy_controller.rb
+++ b/app/controllers/media_proxy_controller.rb
@@ -5,6 +5,8 @@ class MediaProxyController < ApplicationController
 
   skip_before_action :store_current_location
 
+  before_action :authenticate_user!, if: :whitelist_mode?
+
   def show
     RedisLock.acquire(lock_options) do |lock|
       if lock.acquired?
diff --git a/app/controllers/public_timelines_controller.rb b/app/controllers/public_timelines_controller.rb
index 23506b990..324bdc508 100644
--- a/app/controllers/public_timelines_controller.rb
+++ b/app/controllers/public_timelines_controller.rb
@@ -3,7 +3,8 @@
 class PublicTimelinesController < ApplicationController
   layout 'public'
 
-  before_action :check_enabled
+  before_action :authenticate_user!, if: :whitelist_mode?
+  before_action :require_enabled!
   before_action :set_body_classes
   before_action :set_instance_presenter
 
@@ -16,7 +17,7 @@ class PublicTimelinesController < ApplicationController
 
   private
 
-  def check_enabled
+  def require_enabled!
     not_found unless Setting.timeline_preview
   end
 
diff --git a/app/controllers/remote_interaction_controller.rb b/app/controllers/remote_interaction_controller.rb
index cc6993c52..fa742fb0a 100644
--- a/app/controllers/remote_interaction_controller.rb
+++ b/app/controllers/remote_interaction_controller.rb
@@ -5,6 +5,7 @@ class RemoteInteractionController < ApplicationController
 
   layout 'modal'
 
+  before_action :authenticate_user!, if: :whitelist_mode?
   before_action :set_interaction_type
   before_action :set_status
   before_action :set_body_classes
diff --git a/app/controllers/tags_controller.rb b/app/controllers/tags_controller.rb
index d08e5a61a..3cd2d9e20 100644
--- a/app/controllers/tags_controller.rb
+++ b/app/controllers/tags_controller.rb
@@ -8,6 +8,7 @@ class TagsController < ApplicationController
   layout 'public'
 
   before_action :require_signature!, if: -> { request.format == :json && authorized_fetch_mode? }
+  before_action :authenticate_user!, if: :whitelist_mode?
   before_action :set_tag
   before_action :set_body_classes
   before_action :set_instance_presenter
diff --git a/app/helpers/domain_control_helper.rb b/app/helpers/domain_control_helper.rb
index efd328f81..067b2c2cd 100644
--- a/app/helpers/domain_control_helper.rb
+++ b/app/helpers/domain_control_helper.rb
@@ -12,6 +12,14 @@ module DomainControlHelper
       end
     end
 
-    DomainBlock.blocked?(domain)
+    if whitelist_mode?
+      !DomainAllow.allowed?(domain)
+    else
+      DomainBlock.blocked?(domain)
+    end
+  end
+
+  def whitelist_mode?
+    Rails.configuration.x.whitelist_mode
   end
 end
diff --git a/app/models/domain_allow.rb b/app/models/domain_allow.rb
new file mode 100644
index 000000000..85018b636
--- /dev/null
+++ b/app/models/domain_allow.rb
@@ -0,0 +1,33 @@
+# frozen_string_literal: true
+
+# == Schema Information
+#
+# Table name: domain_allows
+#
+#  id         :bigint(8)        not null, primary key
+#  domain     :string           default(""), not null
+#  created_at :datetime         not null
+#  updated_at :datetime         not null
+#
+
+class DomainAllow < ApplicationRecord
+  include DomainNormalizable
+
+  validates :domain, presence: true, uniqueness: true
+
+  scope :matches_domain, ->(value) { where(arel_table[:domain].matches("%#{value}%")) }
+
+  class << self
+    def allowed?(domain)
+      !rule_for(domain).nil?
+    end
+
+    def rule_for(domain)
+      return if domain.blank?
+
+      uri = Addressable::URI.new.tap { |u| u.host = domain.gsub(/[\/]/, '') }
+
+      find_by(domain: uri.normalized_host)
+    end
+  end
+end
diff --git a/app/models/instance.rb b/app/models/instance.rb
index 797a191e0..3c740f8a2 100644
--- a/app/models/instance.rb
+++ b/app/models/instance.rb
@@ -7,8 +7,9 @@ class Instance
 
   def initialize(resource)
     @domain         = resource.domain
-    @accounts_count = resource.is_a?(DomainBlock) ? nil : resource.accounts_count
+    @accounts_count = resource.respond_to?(:accounts_count) ? resource.accounts_count : nil
     @domain_block   = resource.is_a?(DomainBlock) ? resource : DomainBlock.rule_for(domain)
+    @domain_allow   = resource.is_a?(DomainAllow) ? resource : DomainAllow.rule_for(domain)
   end
 
   def countable?
diff --git a/app/models/instance_filter.rb b/app/models/instance_filter.rb
index 848fff53e..8bfab826d 100644
--- a/app/models/instance_filter.rb
+++ b/app/models/instance_filter.rb
@@ -12,6 +12,10 @@ class InstanceFilter
       scope = DomainBlock
       scope = scope.matches_domain(params[:by_domain]) if params[:by_domain].present?
       scope.order(id: :desc)
+    elsif params[:allowed].present?
+      scope = DomainAllow
+      scope = scope.matches_domain(params[:by_domain]) if params[:by_domain].present?
+      scope.order(id: :desc)
     else
       scope = Account.remote
       scope = scope.matches_domain(params[:by_domain]) if params[:by_domain].present?
diff --git a/app/policies/domain_allow_policy.rb b/app/policies/domain_allow_policy.rb
new file mode 100644
index 000000000..5030453bb
--- /dev/null
+++ b/app/policies/domain_allow_policy.rb
@@ -0,0 +1,11 @@
+# frozen_string_literal: true
+
+class DomainAllowPolicy < ApplicationPolicy
+  def create?
+    admin?
+  end
+
+  def destroy?
+    admin?
+  end
+end
diff --git a/app/services/concerns/payloadable.rb b/app/services/concerns/payloadable.rb
index 953740faa..7f9f21c4b 100644
--- a/app/services/concerns/payloadable.rb
+++ b/app/services/concerns/payloadable.rb
@@ -14,6 +14,6 @@ module Payloadable
   end
 
   def signing_enabled?
-    ENV['AUTHORIZED_FETCH'] != 'true'
+    ENV['AUTHORIZED_FETCH'] != 'true' && !Rails.configuration.x.whitelist_mode
   end
 end
diff --git a/app/services/unallow_domain_service.rb b/app/services/unallow_domain_service.rb
new file mode 100644
index 000000000..d4387c1a1
--- /dev/null
+++ b/app/services/unallow_domain_service.rb
@@ -0,0 +1,11 @@
+# frozen_string_literal: true
+
+class UnallowDomainService < BaseService
+  def call(domain_allow)
+    Account.where(domain: domain_allow.domain).find_each do |account|
+      SuspendAccountService.new.call(account, destroy: true)
+    end
+
+    domain_allow.destroy
+  end
+end
diff --git a/app/views/admin/domain_allows/new.html.haml b/app/views/admin/domain_allows/new.html.haml
new file mode 100644
index 000000000..52599857a
--- /dev/null
+++ b/app/views/admin/domain_allows/new.html.haml
@@ -0,0 +1,14 @@
+- content_for :header_tags do
+  = javascript_pack_tag 'admin', integrity: true, async: true, crossorigin: 'anonymous'
+
+- content_for :page_title do
+  = t('admin.domain_allows.add_new')
+
+= simple_form_for @domain_allow, url: admin_domain_allows_path do |f|
+  = render 'shared/error_messages', object: @domain_allow
+
+  .fields-group
+    = f.input :domain, wrapper: :with_label, label: t('admin.domain_blocks.domain'), required: true
+
+  .actions
+    = f.button :button, t('admin.domain_allows.add_new'), type: :submit
diff --git a/app/views/admin/instances/index.html.haml b/app/views/admin/instances/index.html.haml
index 61e578409..982dc5035 100644
--- a/app/views/admin/instances/index.html.haml
+++ b/app/views/admin/instances/index.html.haml
@@ -6,24 +6,30 @@
     %strong= t('admin.instances.moderation.title')
     %ul
       %li= filter_link_to t('admin.instances.moderation.all'), limited: nil
-      %li= filter_link_to t('admin.instances.moderation.limited'), limited: '1'
+
+      - unless whitelist_mode?
+        %li= filter_link_to t('admin.instances.moderation.limited'), limited: '1'
 
   %div{ style: 'flex: 1 1 auto; text-align: right' }
-    = link_to t('admin.domain_blocks.add_new'), new_admin_domain_block_path, class: 'button'
+    - if whitelist_mode?
+      = link_to t('admin.domain_allows.add_new'), new_admin_domain_allow_path, class: 'button'
+    - else
+      = link_to t('admin.domain_blocks.add_new'), new_admin_domain_block_path, class: 'button'
 
-= form_tag admin_instances_url, method: 'GET', class: 'simple_form' do
-  .fields-group
-    - Admin::FilterHelper::INSTANCES_FILTERS.each do |key|
-      - if params[key].present?
-        = hidden_field_tag key, params[key]
+- unless whitelist_mode?
+  = form_tag admin_instances_url, method: 'GET', class: 'simple_form' do
+    .fields-group
+      - Admin::FilterHelper::INSTANCES_FILTERS.each do |key|
+        - if params[key].present?
+          = hidden_field_tag key, params[key]
 
-    - %i(by_domain).each do |key|
-      .input.string.optional
-        = text_field_tag key, params[key], class: 'string optional', placeholder: I18n.t("admin.instances.#{key}")
+      - %i(by_domain).each do |key|
+        .input.string.optional
+          = text_field_tag key, params[key], class: 'string optional', placeholder: I18n.t("admin.instances.#{key}")
 
-    .actions
-      %button= t('admin.accounts.search')
-      = link_to t('admin.accounts.reset'), admin_instances_path, class: 'button negative'
+      .actions
+        %button= t('admin.accounts.search')
+        = link_to t('admin.accounts.reset'), admin_instances_path, class: 'button negative'
 
 %hr.spacer/
 
@@ -47,8 +53,11 @@
               - unless first_item
                 &bull;
               = t('admin.domain_blocks.rejecting_reports')
+          - elsif whitelist_mode?
+            = t('admin.accounts.whitelisted')
           - else
             = t('admin.accounts.no_limits_imposed')
       - if instance.countable?
         .trends__item__current{ title: t('admin.instances.known_accounts', count: instance.accounts_count) }= number_to_human instance.accounts_count, strip_insignificant_zeros: true
+
 = paginate paginated_instances
diff --git a/app/views/admin/instances/show.html.haml b/app/views/admin/instances/show.html.haml
index c7992a490..fbb49ba02 100644
--- a/app/views/admin/instances/show.html.haml
+++ b/app/views/admin/instances/show.html.haml
@@ -38,7 +38,9 @@
     = link_to t('admin.accounts.title'), admin_accounts_path(remote: '1', by_domain: @instance.domain), class: 'button'
 
   %div{ style: 'float: right' }
-    - if @domain_block
+    - if @domain_allow
+      = link_to t('admin.domain_allows.undo'), admin_domain_allow_path(@domain_allow), class: 'button button--destructive', data: { confirm: t('admin.accounts.are_you_sure'), method: :delete }
+    - elsif @domain_block
       = link_to t('admin.domain_blocks.undo'), admin_domain_block_path(@domain_block), class: 'button'
     - else
       = link_to t('admin.domain_blocks.add_new'), new_admin_domain_block_path(_domain: @instance.domain), class: 'button'
diff --git a/app/views/admin/settings/edit.html.haml b/app/views/admin/settings/edit.html.haml
index b3bf3849c..1e2ed3f77 100644
--- a/app/views/admin/settings/edit.html.haml
+++ b/app/views/admin/settings/edit.html.haml
@@ -42,11 +42,12 @@
 
   %hr.spacer/
 
-  .fields-group
-    = f.input :timeline_preview, as: :boolean, wrapper: :with_label, label: t('admin.settings.timeline_preview.title'), hint: t('admin.settings.timeline_preview.desc_html')
+  - unless whitelist_mode?
+    .fields-group
+      = f.input :timeline_preview, as: :boolean, wrapper: :with_label, label: t('admin.settings.timeline_preview.title'), hint: t('admin.settings.timeline_preview.desc_html')
 
-  .fields-group
-    = f.input :show_known_fediverse_at_about_page, as: :boolean, wrapper: :with_label, label: t('admin.settings.show_known_fediverse_at_about_page.title'), hint: t('admin.settings.show_known_fediverse_at_about_page.desc_html')
+    .fields-group
+      = f.input :show_known_fediverse_at_about_page, as: :boolean, wrapper: :with_label, label: t('admin.settings.show_known_fediverse_at_about_page.title'), hint: t('admin.settings.show_known_fediverse_at_about_page.desc_html')
 
   .fields-group
     = f.input :show_staff_badge, as: :boolean, wrapper: :with_label, label: t('admin.settings.show_staff_badge.title'), hint: t('admin.settings.show_staff_badge.desc_html')
@@ -54,17 +55,18 @@
   .fields-group
     = f.input :open_deletion, as: :boolean, wrapper: :with_label, label: t('admin.settings.registrations.deletion.title'), hint: t('admin.settings.registrations.deletion.desc_html')
 
-  .fields-group
-    = f.input :activity_api_enabled, as: :boolean, wrapper: :with_label, label: t('admin.settings.activity_api_enabled.title'), hint: t('admin.settings.activity_api_enabled.desc_html')
+  - unless whitelist_mode?
+    .fields-group
+      = f.input :activity_api_enabled, as: :boolean, wrapper: :with_label, label: t('admin.settings.activity_api_enabled.title'), hint: t('admin.settings.activity_api_enabled.desc_html')
 
-  .fields-group
-    = f.input :peers_api_enabled, as: :boolean, wrapper: :with_label, label: t('admin.settings.peers_api_enabled.title'), hint: t('admin.settings.peers_api_enabled.desc_html')
+    .fields-group
+      = f.input :peers_api_enabled, as: :boolean, wrapper: :with_label, label: t('admin.settings.peers_api_enabled.title'), hint: t('admin.settings.peers_api_enabled.desc_html')
 
-  .fields-group
-    = f.input :preview_sensitive_media, as: :boolean, wrapper: :with_label, label: t('admin.settings.preview_sensitive_media.title'), hint: t('admin.settings.preview_sensitive_media.desc_html')
+    .fields-group
+      = f.input :preview_sensitive_media, as: :boolean, wrapper: :with_label, label: t('admin.settings.preview_sensitive_media.title'), hint: t('admin.settings.preview_sensitive_media.desc_html')
 
-  .fields-group
-    = f.input :profile_directory, as: :boolean, wrapper: :with_label, label: t('admin.settings.profile_directory.title'), hint: t('admin.settings.profile_directory.desc_html')
+    .fields-group
+      = f.input :profile_directory, as: :boolean, wrapper: :with_label, label: t('admin.settings.profile_directory.title'), hint: t('admin.settings.profile_directory.desc_html')
 
   .fields-group
     = f.input :spam_check_enabled, as: :boolean, wrapper: :with_label, label: t('admin.settings.spam_check_enabled.title'), hint: t('admin.settings.spam_check_enabled.desc_html')
@@ -76,7 +78,7 @@
 
   .fields-group
     = f.input :closed_registrations_message, as: :text, wrapper: :with_block_label, label: t('admin.settings.registrations.closed_message.title'), hint: t('admin.settings.registrations.closed_message.desc_html'), input_html: { rows: 8 }
-    = f.input :site_extended_description, wrapper: :with_block_label, as: :text, label: t('admin.settings.site_description_extended.title'), hint: t('admin.settings.site_description_extended.desc_html'), input_html: { rows: 8 }
+    = f.input :site_extended_description, wrapper: :with_block_label, as: :text, label: t('admin.settings.site_description_extended.title'), hint: t('admin.settings.site_description_extended.desc_html'), input_html: { rows: 8 } unless whitelist_mode?
     = f.input :site_terms, wrapper: :with_block_label, as: :text, label: t('admin.settings.site_terms.title'), hint: t('admin.settings.site_terms.desc_html'), input_html: { rows: 8 }
     = f.input :custom_css, wrapper: :with_block_label, as: :text, input_html: { rows: 8 }, label: t('admin.settings.custom_css.title'), hint: t('admin.settings.custom_css.desc_html')
 
diff --git a/app/views/auth/registrations/new.html.haml b/app/views/auth/registrations/new.html.haml
index b4a7cced5..83384d737 100644
--- a/app/views/auth/registrations/new.html.haml
+++ b/app/views/auth/registrations/new.html.haml
@@ -33,7 +33,7 @@
   = f.input :invite_code, as: :hidden
 
   .fields-group
-    = f.input :agreement, as: :boolean, wrapper: :with_label, label: t('auth.checkbox_agreement_html', rules_path: about_more_path, terms_path: terms_path)
+    = f.input :agreement, as: :boolean, wrapper: :with_label, label: whitelist_mode? ? t('auth.checkbox_agreement_without_rules_html', terms_path: terms_path) : t('auth.checkbox_agreement_html', rules_path: about_more_path, terms_path: terms_path)
 
   .actions
     = f.button :button, @invite.present? ? t('auth.register') : sign_up_message, type: :submit
diff --git a/app/views/layouts/public.html.haml b/app/views/layouts/public.html.haml
index 2929ac599..69738a2f7 100644
--- a/app/views/layouts/public.html.haml
+++ b/app/views/layouts/public.html.haml
@@ -10,10 +10,13 @@
             = link_to root_url, class: 'brand' do
               = svg_logo_full
 
-            = link_to t('directories.directory'), explore_path, class: 'nav-link optional' if Setting.profile_directory
-            = link_to t('about.about_this'), about_more_path, class: 'nav-link optional'
-            = link_to t('about.apps'), 'https://joinmastodon.org/apps', class: 'nav-link optional'
+            - unless whitelist_mode?
+              = link_to t('directories.directory'), explore_path, class: 'nav-link optional' if Setting.profile_directory
+              = link_to t('about.about_this'), about_more_path, class: 'nav-link optional'
+              = link_to t('about.apps'), 'https://joinmastodon.org/apps', class: 'nav-link optional'
+
           .nav-center
+
           .nav-right
             - if user_signed_in?
               = link_to t('settings.back'), root_url, class: 'nav-link nav-button webapp-btn'
diff --git a/config/initializers/2_whitelist_mode.rb b/config/initializers/2_whitelist_mode.rb
new file mode 100644
index 000000000..a17ad07a2
--- /dev/null
+++ b/config/initializers/2_whitelist_mode.rb
@@ -0,0 +1,5 @@
+# frozen_string_literal: true
+
+Rails.application.configure do
+  config.x.whitelist_mode = ENV['WHITELIST_MODE'] == 'true'
+end
diff --git a/config/locales/en.yml b/config/locales/en.yml
index 9e1be87be..6c1a34300 100644
--- a/config/locales/en.yml
+++ b/config/locales/en.yml
@@ -186,6 +186,7 @@ en:
       username: Username
       warn: Warn
       web: Web
+      whitelisted: Whitelisted
     action_logs:
       actions:
         assigned_to_self_report: "%{name} assigned report %{target} to themselves"
@@ -269,6 +270,11 @@ en:
       week_interactions: interactions this week
       week_users_active: active this week
       week_users_new: users this week
+    domain_allows:
+      add_new: Whitelist domain
+      created_msg: Domain has been successfully whitelisted
+      destroyed_msg: Domain has been removed from the whitelist
+      undo: Remove from whitelist
     domain_blocks:
       add_new: Add new domain block
       created_msg: Domain block is now being processed
@@ -524,6 +530,7 @@ en:
     apply_for_account: Request an invite
     change_password: Password
     checkbox_agreement_html: I agree to the <a href="%{rules_path}" target="_blank">server rules</a> and <a href="%{terms_path}" target="_blank">terms of service</a>
+    checkbox_agreement_without_rules_html: I agree to the <a href="%{terms_path}" target="_blank">terms of service</a>
     delete_account: Delete account
     delete_account_html: If you wish to delete your account, you can <a href="%{path}">proceed here</a>. You will be asked for confirmation.
     didnt_get_confirmation: Didn't receive confirmation instructions?
diff --git a/config/locales/simple_form.en.yml b/config/locales/simple_form.en.yml
index 12a7ec2b3..10b30e627 100644
--- a/config/locales/simple_form.en.yml
+++ b/config/locales/simple_form.en.yml
@@ -38,6 +38,8 @@ en:
         setting_use_pending_items: Hide timeline updates behind a click instead of automatically scrolling the feed
         username: Your username will be unique on %{domain}
         whole_word: When the keyword or phrase is alphanumeric only, it will only be applied if it matches the whole word
+      domain_allow:
+        domain: This domain will be able to fetch data from this server and incoming data from it will be processed and stored
       featured_tag:
         name: 'You might want to use one of these:'
       imports:
diff --git a/config/navigation.rb b/config/navigation.rb
index 5ab2e4399..9b46da603 100644
--- a/config/navigation.rb
+++ b/config/navigation.rb
@@ -39,7 +39,7 @@ SimpleNavigation::Configuration.run do |navigation|
       s.item :accounts, safe_join([fa_icon('users fw'), t('admin.accounts.title')]), admin_accounts_url, highlights_on: %r{/admin/accounts|/admin/pending_accounts}
       s.item :invites, safe_join([fa_icon('user-plus fw'), t('admin.invites.title')]), admin_invites_path
       s.item :tags, safe_join([fa_icon('tag fw'), t('admin.tags.title')]), admin_tags_path
-      s.item :instances, safe_join([fa_icon('cloud fw'), t('admin.instances.title')]), admin_instances_url(limited: '1'), highlights_on: %r{/admin/instances|/admin/domain_blocks}, if: -> { current_user.admin? }
+      s.item :instances, safe_join([fa_icon('cloud fw'), t('admin.instances.title')]), admin_instances_url(limited: whitelist_mode? ? nil : '1'), highlights_on: %r{/admin/instances|/admin/domain_blocks|/admin/domain_allows}, if: -> { current_user.admin? }
       s.item :email_domain_blocks, safe_join([fa_icon('envelope fw'), t('admin.email_domain_blocks.title')]), admin_email_domain_blocks_url, highlights_on: %r{/admin/email_domain_blocks}, if: -> { current_user.admin? }
     end
 
diff --git a/config/routes.rb b/config/routes.rb
index b6c215888..04424bbbd 100644
--- a/config/routes.rb
+++ b/config/routes.rb
@@ -154,6 +154,7 @@ Rails.application.routes.draw do
   namespace :admin do
     get '/dashboard', to: 'dashboard#index'
 
+    resources :domain_allows, only: [:new, :create, :show, :destroy]
     resources :domain_blocks, only: [:new, :create, :show, :destroy]
     resources :email_domain_blocks, only: [:index, :new, :create, :destroy]
     resources :action_logs, only: [:index]
diff --git a/db/migrate/20190705002136_create_domain_allows.rb b/db/migrate/20190705002136_create_domain_allows.rb
new file mode 100644
index 000000000..83b0728d9
--- /dev/null
+++ b/db/migrate/20190705002136_create_domain_allows.rb
@@ -0,0 +1,9 @@
+class CreateDomainAllows < ActiveRecord::Migration[5.2]
+  def change
+    create_table :domain_allows do |t|
+      t.string :domain, default: '', null: false, index: { unique: true }
+
+      t.timestamps
+    end
+  end
+end
diff --git a/db/schema.rb b/db/schema.rb
index 1847305c7..2d83d8b76 100644
--- a/db/schema.rb
+++ b/db/schema.rb
@@ -10,7 +10,7 @@
 #
 # It's strongly recommended that you check this file into your version control system.
 
-ActiveRecord::Schema.define(version: 2019_07_26_175042) do
+ActiveRecord::Schema.define(version: 2019_07_28_084117) do
 
   # These are extensions that must be enabled in order to support this database
   enable_extension "plpgsql"
@@ -245,6 +245,13 @@ ActiveRecord::Schema.define(version: 2019_07_26_175042) do
     t.index ["account_id"], name: "index_custom_filters_on_account_id"
   end
 
+  create_table "domain_allows", force: :cascade do |t|
+    t.string "domain", default: "", null: false
+    t.datetime "created_at", null: false
+    t.datetime "updated_at", null: false
+    t.index ["domain"], name: "index_domain_allows_on_domain", unique: true
+  end
+
   create_table "domain_blocks", force: :cascade do |t|
     t.string "domain", default: "", null: false
     t.datetime "created_at", null: false
diff --git a/lib/mastodon/domains_cli.rb b/lib/mastodon/domains_cli.rb
index b081581fe..f30062363 100644
--- a/lib/mastodon/domains_cli.rb
+++ b/lib/mastodon/domains_cli.rb
@@ -12,17 +12,33 @@ module Mastodon
     end
 
     option :dry_run, type: :boolean
-    desc 'purge DOMAIN', 'Remove accounts from a DOMAIN without a trace'
+    option :whitelist_mode, type: :boolean
+    desc 'purge [DOMAIN]', 'Remove accounts from a DOMAIN without a trace'
     long_desc <<-LONG_DESC
       Remove all accounts from a given DOMAIN without leaving behind any
       records. Unlike a suspension, if the DOMAIN still exists in the wild,
       it means the accounts could return if they are resolved again.
+
+      When the --whitelist-mode option is given, instead of purging accounts
+      from a single domain, all accounts from domains that are not whitelisted
+      are removed from the database.
     LONG_DESC
-    def purge(domain)
+    def purge(domain = nil)
       removed = 0
       dry_run = options[:dry_run] ? ' (DRY RUN)' : ''
 
-      Account.where(domain: domain).find_each do |account|
+      scope = begin
+        if options[:whitelist_mode]
+          Account.remote.where.not(domain: DomainAllow.pluck(:domain))
+        elsif domain.present?
+          Account.remote.where(domain: domain)
+        else
+          say('No domain given', :red)
+          exit(1)
+        end
+      end
+
+      scope.find_each do |account|
         SuspendAccountService.new.call(account, destroy: true) unless options[:dry_run]
         removed += 1
         say('.', :green, false)
diff --git a/spec/fabricators/domain_allow_fabricator.rb b/spec/fabricators/domain_allow_fabricator.rb
new file mode 100644
index 000000000..6226b1e20
--- /dev/null
+++ b/spec/fabricators/domain_allow_fabricator.rb
@@ -0,0 +1,3 @@
+Fabricator(:domain_allow) do
+  domain "MyString"
+end
diff --git a/spec/models/domain_allow_spec.rb b/spec/models/domain_allow_spec.rb
new file mode 100644
index 000000000..e65435127
--- /dev/null
+++ b/spec/models/domain_allow_spec.rb
@@ -0,0 +1,5 @@
+require 'rails_helper'
+
+RSpec.describe DomainAllow, type: :model do
+  pending "add some examples to (or delete) #{__FILE__}"
+end
diff --git a/streaming/index.js b/streaming/index.js
index 0529804b1..304e7e046 100644
--- a/streaming/index.js
+++ b/streaming/index.js
@@ -12,6 +12,7 @@ const uuid = require('uuid');
 const fs = require('fs');
 
 const env = process.env.NODE_ENV || 'development';
+const alwaysRequireAuth = process.env.WHITELIST_MODE === 'true' || process.env.AUTHORIZED_FETCH === 'true';
 
 dotenv.config({
   path: env === 'production' ? '.env.production' : '.env',
@@ -271,7 +272,7 @@ const startWorker = (workerId) => {
 
   const wsVerifyClient = (info, cb) => {
     const location = url.parse(info.req.url, true);
-    const authRequired = !PUBLIC_STREAMS.some(stream => stream === location.query.stream);
+    const authRequired = alwaysRequireAuth || !PUBLIC_STREAMS.some(stream => stream === location.query.stream);
     const allowedScopes = [];
 
     if (authRequired) {
@@ -306,7 +307,7 @@ const startWorker = (workerId) => {
       return;
     }
 
-    const authRequired = !PUBLIC_ENDPOINTS.some(endpoint => endpoint === req.path);
+    const authRequired = alwaysRequireAuth || !PUBLIC_ENDPOINTS.some(endpoint => endpoint === req.path);
     const allowedScopes = [];
 
     if (authRequired) {
-- 
cgit